Implement docker networks. Fixes #1982.

Update test_system_network.py

Black formatting

Prevents the use of 'host', 'bridge', and 'null' as connection names.

Update test_system_network.py following implementation of docker networking.

Check if rock-on uses host network and disable network customization if true.

move existing & proposed low level docker functions to system/docker.py
Helps to remove the possibility of circular imports
Move existing docker_status from rockon_helpers to system/docker
Improves separation of concerns between system level docker and Rock-ons
Establish 'sysnet' (system/network) import name space within views

Display rocknet details in Network summary table.
Use docker_name for rocknets in network dashboard widget:
Get containers and corresponding rock-on name into the connection's docker_options.
Get Rocknet(s) for rockons:
- Implement Rocknet Backbone Model: RockOnNetwork and its Collection
- Create API call to fetch data from DContainerNetwork through RockOnNetwork
- Display currently-attached rocknet(s) in rockon summary table

Allow new networks to be specified directly from the field.
Changes to DPort model:
- add publish field
- add container_name property method

Join Networks:
- Switch to sequential fetching of Ports, Containers, and then Networks as the latter need to be stopping.
- Display user networks as multi-select field per containers in rock-on
Unpublish Ports UI: Add backend logic to skip port at docker run time if unpublished
Disable UI button if UI-port is unpublished
Implement UI to Add, Edit, and Delete docker networks.
Disallow connection toggle and edit for docker networks.
Display docker_name in network summary table.
Fix for Container_links: Create and connect each linked container to a given network defined in the Rock-on JSON definition file.

src/rockstor/cli/disks_console.py

@@ -90,6 +90,6 @@ def do_wipe(self, args):
     def help_wipe(self):
         snps = "Wipe the partition table of a disk"
         params = {
-            "disk_name": "Name of the disk to be wiped of it's data",
+            "disk_name": "Name of the disk to be wiped of its data",
         }
         self.print_help(snps, "wipe", args=("disk_name",), params=params)

src/rockstor/storageadmin/migrations/0013_auto_20200815_2004.py

@@ -0,0 +1,52 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('storageadmin', '0012_auto_20200429_1428'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='BridgeConnection',
+            fields=[
+                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
+                ('docker_name', models.CharField(max_length=64, null=True)),
+                ('usercon', models.BooleanField(default=False)),
+                ('aux_address', models.CharField(max_length=2048, null=True)),
+                ('dgateway', models.CharField(max_length=64, null=True)),
+                ('host_binding', models.CharField(max_length=64, null=True)),
+                ('icc', models.BooleanField(default=False)),
+                ('internal', models.BooleanField(default=False)),
+                ('ip_masquerade', models.BooleanField(default=False)),
+                ('ip_range', models.CharField(max_length=64, null=True)),
+                ('subnet', models.CharField(max_length=64, null=True)),
+                ('connection', models.ForeignKey(to='storageadmin.NetworkConnection', null=True)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='DContainerNetwork',
+            fields=[
+                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
+                ('connection', models.ForeignKey(to='storageadmin.BridgeConnection')),
+                ('container', models.ForeignKey(to='storageadmin.DContainer')),
+            ],
+        ),
+        migrations.AddField(
+            model_name='dport',
+            name='publish',
+            field=models.BooleanField(default=True),
+        ),
+        migrations.AlterUniqueTogether(
+            name='dcontainerlink',
+            unique_together=set([('source', 'destination', 'name')]),
+        ),
+        migrations.AlterUniqueTogether(
+            name='dcontainernetwork',
+            unique_together=set([('container', 'connection')]),
+        ),
+    ]

src/rockstor/storageadmin/models/__init__.py

@@ -24,13 +24,9 @@
 from nfs_export import NFSExport  # noqa E501
 from iscsi_target import IscsiTarget  # noqa E501
 from api_keys import APIKeys  # noqa E501
-from network_interface import (
-    NetworkConnection,
-    NetworkDevice,  # noqa E501
-    EthernetConnection,
-    TeamConnection,
-    BondConnection,
-)  # noqa E501
+from network_interface import (NetworkConnection, NetworkDevice,  # noqa E501
+                               EthernetConnection, TeamConnection, BondConnection,
+                               BridgeConnection)  # noqa E501
 from appliance import Appliance  # noqa E501
 from support_case import SupportCase  # noqa E501
 from dashboard_config import DashboardConfig  # noqa E501
@@ -47,30 +43,13 @@
 from oauth_app import OauthApp  # noqa E501
 from pool_balance import PoolBalance  # noqa E501
 from tls_certificate import TLSCertificate  # noqa E501
-from rockon import (
-    RockOn,
-    DImage,
-    DContainer,
-    DPort,
-    DVolume,  # noqa E501
-    ContainerOption,
-    DCustomConfig,
-    DContainerLink,  # noqa E501
-    DContainerEnv,
-    DContainerDevice,
-    DContainerArgs,
-    DContainerLabel,
-)  # noqa E501
-from smart import (
-    SMARTAttribute,
-    SMARTCapability,
-    SMARTErrorLog,  # noqa E501
-    SMARTErrorLogSummary,
-    SMARTTestLog,
-    SMARTTestLogDetail,  # noqa E501
-    SMARTIdentity,
-    SMARTInfo,
-)  # noqa E501
+from rockon import (RockOn, DImage, DContainer, DPort, DVolume,  # noqa E501
+                    ContainerOption, DCustomConfig, DContainerLink,  # noqa E501
+                    DContainerEnv, DContainerDevice, DContainerArgs,
+                    DContainerLabel, DContainerNetwork)  # noqa E501
+from smart import (SMARTAttribute, SMARTCapability, SMARTErrorLog,  # noqa E501
+                   SMARTErrorLogSummary, SMARTTestLog, SMARTTestLogDetail,  # noqa E501
+                   SMARTIdentity, SMARTInfo)  # noqa E501
 from config_backup import ConfigBackup  # noqa E501
 from email import EmailClient  # noqa E501
 from update_subscription import UpdateSubscription  # noqa E501

src/rockstor/storageadmin/models/network_interface.py

@@ -19,7 +19,6 @@
 import json
 from django.db import models
 
-
 # This is the key abstraction for network configuration that is user
 # configurable in Rockstor.  user can add, delete or modify connections which
 # results in CRUD ops on this model and also on other models linked to this
@@ -86,6 +85,8 @@ def ctype(self):
             return "team"
         if self.bondconnection_set.count() > 0:
             return "bond"
+        if self.bridgeconnection_set.count() > 0:
+            return "bridge"
         return None
 
     @property
@@ -112,6 +113,66 @@ def bond_profile(self):
         finally:
             return profile
 
+    @property
+    def docker_name(self):
+        dname = None
+        if self.bridgeconnection_set.count() > 0:
+            brco = self.bridgeconnection_set.first()
+            dname = brco.docker_name
+        return dname
+
+    @property
+    def user_dnet(self):
+        """
+        Returns True if the docker network is a rocknet (defined by the user).
+        Used by rockons.js to list available rocknets available for connection.
+        :return: Boolean
+        """
+        user_dnet = None
+        if self.bridgeconnection_set.count() > 0:
+            brco = self.bridgeconnection_set.first()
+            user_dnet = brco.usercon
+            if user_dnet:
+                user_dnet = True
+        return user_dnet
+
+    @property
+    def docker_options(self):
+        """
+        Gather all connection's settings in a dict to be displayed in the UI connection form
+        needed to edit an existing docker network connection.
+        :return:
+        """
+        docker_options = {}
+        if self.bridgeconnection_set.count() > 0:
+            brco = self.bridgeconnection_set.first()
+            connected_containers = []
+            if brco.dcontainernetwork_set.filter(connection=brco.id).count() > 0:
+                for i in range(
+                    brco.dcontainernetwork_set.filter(connection=brco.id).count()
+                ):
+                    cname = (
+                        brco.dcontainernetwork_set.filter(connection=brco.id)
+                        .order_by("id")[i]
+                        .container_name
+                    )
+                    rname = (
+                        brco.dcontainernetwork_set.filter(connection=brco.id)
+                        .order_by("id")[i]
+                        .container.rockon.name
+                    )
+                    connected_containers.append("{} ({})".format(cname, rname))
+            docker_options["aux_address"] = brco.aux_address
+            docker_options["dgateway"] = brco.dgateway
+            docker_options["host_binding"] = brco.host_binding
+            docker_options["icc"] = brco.icc
+            docker_options["internal"] = brco.internal
+            docker_options["ip_masquerade"] = brco.ip_masquerade
+            docker_options["ip_range"] = brco.ip_range
+            docker_options["subnet"] = brco.subnet
+            docker_options["containers"] = connected_containers
+        return docker_options
+
     class Meta:
         app_label = "storageadmin"
 
@@ -141,6 +202,17 @@ def cname(self):
             return None
         return self.connection.name
 
+    @property
+    def dev_name(self):
+        """
+        Return the user-friendly docker_name as device name for bridge connections
+        to be displayed in the network widget on the dashboard.
+        :return:
+        """
+        if (self.dtype == "bridge") and (self.connection is not None):
+            return self.connection.docker_name
+        return self.name
+
     class Meta:
         app_label = "storageadmin"
 
@@ -177,3 +249,20 @@ class BondConnection(models.Model):
 
     class Meta:
         app_label = "storageadmin"
+
+
+class BridgeConnection(models.Model):
+    connection = models.ForeignKey(NetworkConnection, null=True)
+    docker_name = models.CharField(max_length=64, null=True)
+    usercon = models.BooleanField(default=False)
+    aux_address = models.CharField(max_length=2048, null=True)
+    dgateway = models.CharField(max_length=64, null=True)
+    host_binding = models.CharField(max_length=64, null=True)
+    icc = models.BooleanField(default=False)
+    internal = models.BooleanField(default=False)
+    ip_masquerade = models.BooleanField(default=False)
+    ip_range = models.CharField(max_length=64, null=True)
+    subnet = models.CharField(max_length=64, null=True)
+
+    class Meta:
+        app_label = "storageadmin"

src/rockstor/storageadmin/models/rockon.py

@@ -17,7 +17,9 @@
 """
 
 from django.db import models
-from storageadmin.models import Share
+from storageadmin.models import Share, BridgeConnection
+
+from system.docker import probe_running_containers
 
 
 class RockOn(models.Model):
@@ -44,6 +46,36 @@ def ui_port(self):
                     return po.hostp
         return None
 
+    @property
+    def ui_publish(self):
+        """
+        Returns True if the rock-on has a container with a UI port defined
+        and set to be published. This property is used to decide whether or
+        not to disable a rock-on's UI button.
+        :return:
+        """
+        if not self.ui:
+            return None
+        for co in self.dcontainer_set.all():
+            for po in co.dport_set.all():
+                if po.uiport and po.publish:
+                    return True
+        return None
+
+    @property
+    def host_network(self):
+        """
+        Checks whether the rock-on uses host networking and disable networking
+        post-install customization options accordingly.
+        :return: True if using host networking.
+        """
+        for co in self.dcontainer_set.all():
+            res = probe_running_containers(container=co.name, network="host", all=True)
+            if len(res) > 1:
+                return True
+            else:
+                return False
+
     class Meta:
         app_label = "storageadmin"
 
@@ -77,7 +109,28 @@ class DContainerLink(models.Model):
     name = models.CharField(max_length=64, null=True)
 
     class Meta:
-        unique_together = ("destination", "name")
+        unique_together = ("source", "destination", "name")
+        app_label = "storageadmin"
+
+
+class DContainerNetwork(models.Model):
+    container = models.ForeignKey(DContainer)
+    connection = models.ForeignKey(BridgeConnection)
+
+    @property
+    def docker_name(self):
+        if self.connection is not None:
+            return self.connection.docker_name
+        return None
+
+    @property
+    def container_name(self):
+        if self.container is not None:
+            return self.container.name
+        return None
+
+    class Meta:
+        unique_together = ("container", "connection")
         app_label = "storageadmin"
 
 
@@ -90,6 +143,13 @@ class DPort(models.Model):
     protocol = models.CharField(max_length=32, null=True)
     uiport = models.BooleanField(default=False)
     label = models.CharField(max_length=1024, null=True)
+    publish = models.BooleanField(default=True)
+
+    @property
+    def container_name(self):
+        if self.container is not None:
+            return self.container.name
+        return None
 
     class Meta:
         unique_together = (

src/rockstor/storageadmin/serializers.py

@@ -49,6 +49,7 @@
     DContainerEnv,
     DContainerDevice,
     DContainerLabel,
+    DContainerNetwork,
     SMARTAttribute,
     SMARTCapability,
     SMARTInfo,
@@ -201,6 +202,7 @@ class Meta:
 
 class NetworkDeviceSerializer(serializers.ModelSerializer):
     cname = serializers.CharField()
+    dev_name = serializers.CharField()
 
     class Meta:
         model = NetworkDevice
@@ -211,6 +213,9 @@ class NetworkConnectionSerializer(serializers.ModelSerializer):
     mtu = serializers.IntegerField()
     team_profile = serializers.CharField()
     bond_profile = serializers.CharField()
+    docker_name = serializers.CharField()
+    user_dnet = serializers.BooleanField()
+    docker_options = serializers.DictField()
 
     class Meta:
         model = NetworkConnection
@@ -254,6 +259,8 @@ class Meta:
 
 class RockOnSerializer(serializers.ModelSerializer):
     ui_port = serializers.IntegerField()
+    ui_publish = serializers.BooleanField()
+    host_network = serializers.BooleanField()
 
     class Meta:
         model = RockOn
@@ -272,6 +279,8 @@ class Meta:
 
 
 class RockOnPortSerializer(serializers.ModelSerializer):
+    container_name = serializers.CharField()
+
     class Meta:
         model = DPort
 
@@ -296,6 +305,14 @@ class Meta:
         model = DContainerLabel
 
 
+class RockOnNetworkSerializer(serializers.ModelSerializer):
+    docker_name = serializers.CharField()
+    container_name = serializers.CharField()
+
+    class Meta:
+        model = DContainerNetwork
+
+
 class SMARTCapabilitySerializer(serializers.ModelSerializer):
     class Meta:
         model = SMARTCapability