Merge pull request #121 from networktocode/develop

Sync develop with master

.github/CODEOWNERS

@@ -0,0 +1,2 @@
+# Default owners for all files in this repository
+* @dgarros @mzbroch

.travis.yml

@@ -5,11 +5,11 @@ python:
 - '3.7'
 - '3.8'
 env:
-# Each version of NetBox listed here must have a corresponding directory/configuration file 
-# under development/netbox_<NETBOX_VER>/configuration.py
+# Each version of NetBox listed here uses configuration stored in development/configuration.py
   matrix:
-  - NETBOX_VER=v2.8.3
-  - NETBOX_VER=master
+  - NETBOX_VER=v2.8.9
+  - NETBOX_VER=v2.9.11
+  - NETBOX_VER=v2.10.2
 # Encrypted value for PYPI_TOKEN, this secret has been generated with the following command
 #   travis encrypt PYPI_TOKEN=<value> --add env.global --com
 # Might need to update it once the repo is publish (travis-ci.org vs travis-ci.com)

README.md

@@ -26,9 +26,13 @@ pip install ntc-netbox-plugin-onboarding
 systemctl restart netbox netbox-rq
 ```
 
-> The ntc-netbox-plugin-onboarding v1.3 is compatible with NetBox 2.8
+### Compatibility Matrix
 
-> The ntc-netbox-plugin-onboarding v2 is compatible with NetBox 2.8 and NetBox 2.9
+|                       | Netbox 2.8 | Netbox 2.9 | Netbox 2.10 |
+|-----------------------|------------|------------|-------------|
+| Onboarding Plugin 1.3 |      X     |            |             |
+| Onboarding Plugin 2.0 |      X     |      X     |             |
+| Onboarding Plugin 2.1 |      X     |      X     |      X      |
 
 To ensure NetBox Onboarding plugin is automatically re-installed during future upgrades, create a file named `local_requirements.txt` (if not already existing) in the NetBox root directory (alongside `requirements.txt`) and list the `ntc-netbox-plugin-onboarding` package:
 
@@ -85,6 +89,10 @@ currently two strategies, strict and loose. Strict has to be a direct match, nor
 using a slug. Loose allows a range of search criteria to match a single object. If multiple
 objects are returned an error is raised. 
 
+## Upgrades
+
+When a new release comes out it may be necessary to run a migration of the database to account for any changes in the data models used by this plugin. Execute the command `python3 manage.py migrate`from the NetBox install `netbox/` directory after updating the package.
+
 ## Usage
 
 ### Preparation

development/Dockerfile

@@ -2,7 +2,6 @@
 ARG python_ver=3.7
 FROM python:${python_ver}
 
-ARG netbox_ver=master
 ENV PYTHONUNBUFFERED 1
 
 RUN mkdir -p /opt
@@ -15,6 +14,7 @@ RUN pip install --upgrade pip\
 # -------------------------------------------------------------------------------------
 # Remove redis==3.4.1 from the requirements.txt file as a workaround to #4910
 # https://github.com/netbox-community/netbox/issues/4910, required for version 2.8.8 and earlier
+ARG netbox_ver=master
 RUN git clone --single-branch --branch ${netbox_ver} https://github.com/netbox-community/netbox.git /opt/netbox/ && \
     cd /opt/netbox/ && \
     sed -i '/^redis\=\=/d' /opt/netbox/requirements.txt && \

development/base_configuration.py → development/configuration.py

@@ -1,5 +1,51 @@
-"""NetBox configuration file."""
+"""NetBox configuration."""
 import os
+from distutils.util import strtobool
+from packaging import version
+from django.core.exceptions import ImproperlyConfigured
+from .settings import VERSION  # pylint: disable=relative-beyond-top-level
+
+
+NETBOX_RELEASE_CURRENT = version.parse(VERSION)
+NETBOX_RELEASE_28 = version.parse("2.8")
+NETBOX_RELEASE_29 = version.parse("2.9")
+NETBOX_RELEASE_211 = version.parse("2.11")
+
+# Enforce required configuration parameters
+for key in [
+    "ALLOWED_HOSTS",
+    "POSTGRES_DB",
+    "POSTGRES_USER",
+    "POSTGRES_HOST",
+    "POSTGRES_PASSWORD",
+    "REDIS_HOST",
+    "REDIS_PASSWORD",
+    "SECRET_KEY",
+]:
+    if not os.environ.get(key):
+        raise ImproperlyConfigured(f"Required environment variable {key} is missing.")
+
+
+def is_truthy(arg):
+    """Convert "truthy" strings into Booleans.
+
+    Examples:
+        >>> is_truthy('yes')
+        True
+    Args:
+        arg (str): Truthy string (True values are y, yes, t, true, on and 1; false values are n, no,
+        f, false, off and 0. Raises ValueError if val is anything else.
+    """
+    if isinstance(arg, bool):
+        return arg
+
+    try:
+        bool_val = strtobool(arg)
+    except ValueError:
+        raise ImproperlyConfigured(f"Unexpected variable value: {arg}")  # pylint: disable=raise-missing-from
+
+    return bool(bool_val)
+
 
 # For reference see http://netbox.readthedocs.io/en/latest/configuration/mandatory-settings/
 # Based on https://github.com/digitalocean/netbox/blob/develop/netbox/netbox/configuration.example.py
@@ -16,46 +62,52 @@
 # access to the server via any other hostnames. The first FQDN in the list will be treated as the preferred name.
 #
 # Example: ALLOWED_HOSTS = ['netbox.example.com', 'netbox.internal.local']
-ALLOWED_HOSTS = os.environ.get("ALLOWED_HOSTS", "").split(" ")
+ALLOWED_HOSTS = os.environ["ALLOWED_HOSTS"].split(" ")
 
 # PostgreSQL database configuration.
 DATABASE = {
-    "NAME": os.environ.get("DB_NAME", "netbox"),  # Database name
-    "USER": os.environ.get("DB_USER", ""),  # PostgreSQL username
-    "PASSWORD": os.environ.get("DB_PASSWORD", ""),
+    "NAME": os.environ["POSTGRES_DB"],  # Database name
+    "USER": os.environ["POSTGRES_USER"],  # PostgreSQL username
+    "PASSWORD": os.environ["POSTGRES_PASSWORD"],
     # PostgreSQL password
-    "HOST": os.environ.get("DB_HOST", "localhost"),  # Database server
-    "PORT": os.environ.get("DB_PORT", ""),  # Database port (leave blank for default)
+    "HOST": os.environ["POSTGRES_HOST"],  # Database server
+    "PORT": 5432 if "POSTGRES_PORT" not in os.environ else int(os.environ["POSTGRES_PORT"]),  # Database port
 }
 
 # This key is used for secure generation of random numbers and strings. It must never be exposed outside of this file.
 # For optimal security, SECRET_KEY should be at least 50 characters in length and contain a mix of letters, numbers, and
 # symbols. NetBox will not run without this defined. For more information, see
 # https://docs.djangoproject.com/en/dev/ref/settings/#std:setting-SECRET_KEY
-SECRET_KEY = os.environ.get("SECRET_KEY", "")
+SECRET_KEY = os.environ["SECRET_KEY"]
 
 # Redis database settings. The Redis database is used for caching and background processing such as webhooks
 # Seperate sections for webhooks and caching allow for connecting to seperate Redis instances/datbases if desired.
 # Full connection details are required in both sections, even if they are the same.
 REDIS = {
     "caching": {
-        "HOST": os.environ.get("REDIS_HOST", "redis"),
+        "HOST": os.environ["REDIS_HOST"],
         "PORT": int(os.environ.get("REDIS_PORT", 6379)),
-        "PASSWORD": os.environ.get("REDIS_PASSWORD", ""),
+        "PASSWORD": os.environ["REDIS_PASSWORD"],
         "DATABASE": 1,
-        "DEFAULT_TIMEOUT": 300,
-        "SSL": bool(os.environ.get("REDIS_SSL", False)),
+        "SSL": is_truthy(os.environ.get("REDIS_SSL", False)),
     },
     "tasks": {
-        "HOST": os.environ.get("REDIS_HOST", "redis"),
+        "HOST": os.environ["REDIS_HOST"],
         "PORT": int(os.environ.get("REDIS_PORT", 6379)),
-        "PASSWORD": os.environ.get("REDIS_PASSWORD", ""),
+        "PASSWORD": os.environ["REDIS_PASSWORD"],
         "DATABASE": 0,
-        "DEFAULT_TIMEOUT": 300,
-        "SSL": bool(os.environ.get("REDIS_SSL", False)),
+        "SSL": is_truthy(os.environ.get("REDIS_SSL", False)),
     },
 }
 
+if NETBOX_RELEASE_28 < NETBOX_RELEASE_CURRENT < NETBOX_RELEASE_29:
+    # NetBox 2.8.x Specific Settings
+    REDIS["caching"]["DEFAULT_TIMEOUT"] = 300
+    REDIS["tasks"]["DEFAULT_TIMEOUT"] = 300
+elif NETBOX_RELEASE_CURRENT < NETBOX_RELEASE_211:
+    RQ_DEFAULT_TIMEOUT = 300
+else:
+    raise ImproperlyConfigured(f"Version {NETBOX_RELEASE_CURRENT} of NetBox is unsupported at this time.")
 
 #########################
 #                       #
@@ -71,8 +123,8 @@
 
 # Optionally display a persistent banner at the top and/or bottom of every page. HTML is allowed. To display the same
 # content in both banners, define BANNER_TOP and set BANNER_BOTTOM = BANNER_TOP.
-BANNER_TOP = os.environ.get("BANNER_TOP", None)
-BANNER_BOTTOM = os.environ.get("BANNER_BOTTOM", None)
+BANNER_TOP = os.environ.get("BANNER_TOP", "")
+BANNER_BOTTOM = os.environ.get("BANNER_BOTTOM", "")
 
 # Text to include on the login page above the login form. HTML is allowed.
 BANNER_LOGIN = os.environ.get("BANNER_LOGIN", "")
@@ -87,58 +139,65 @@
 # API Cross-Origin Resource Sharing (CORS) settings. If CORS_ORIGIN_ALLOW_ALL is set to True, all origins will be
 # allowed. Otherwise, define a list of allowed origins using either CORS_ORIGIN_WHITELIST or
 # CORS_ORIGIN_REGEX_WHITELIST. For more information, see https://github.com/ottoyiu/django-cors-headers
-CORS_ORIGIN_ALLOW_ALL = True
+CORS_ORIGIN_ALLOW_ALL = is_truthy(os.environ.get("CORS_ORIGIN_ALLOW_ALL", False))
 CORS_ORIGIN_WHITELIST = []
 CORS_ORIGIN_REGEX_WHITELIST = []
 
 # Set to True to enable server debugging. WARNING: Debugging introduces a substantial performance penalty and may reveal
 # sensitive information about your installation. Only enable debugging while performing testing. Never enable debugging
 # on a production system.
-DEBUG = True
-DEVELOPER = True
+DEBUG = is_truthy(os.environ.get("DEBUG", False))
+DEVELOPER = is_truthy(os.environ.get("DEVELOPER", False))
 
 # Email settings
 EMAIL = {
-    "SERVER": "localhost",
-    "PORT": 25,
-    "USERNAME": "",
-    "PASSWORD": "",
-    "TIMEOUT": 10,
-    "FROM_EMAIL": "",
+    "SERVER": os.environ.get("EMAIL_SERVER", "localhost"),
+    "PORT": int(os.environ.get("EMAIL_PORT", 25)),
+    "USERNAME": os.environ.get("EMAIL_USERNAME", ""),
+    "PASSWORD": os.environ.get("EMAIL_PASSWORD", ""),
+    "TIMEOUT": int(os.environ.get("EMAIL_TIMEOUT", 10)),  # seconds
+    "FROM_EMAIL": os.environ.get("EMAIL_FROM", ""),
 }
 
 # Enforcement of unique IP space can be toggled on a per-VRF basis.
 # To enforce unique IP space within the global table (all prefixes and IP addresses not assigned to a VRF),
 # set ENFORCE_GLOBAL_UNIQUE to True.
-ENFORCE_GLOBAL_UNIQUE = False
+ENFORCE_GLOBAL_UNIQUE = is_truthy(os.environ.get("ENFORCE_GLOBAL_UNIQUE", False))
+
+# HTTP proxies NetBox should use when sending outbound HTTP requests (e.g. for webhooks).
+# HTTP_PROXIES = {
+#     "http": "http://192.0.2.1:3128",
+#     "https": "http://192.0.2.1:1080",
+# }
+
+# IP addresses recognized as internal to the system. The debugging toolbar will be available only to clients accessing
+# NetBox from an internal IP.
+INTERNAL_IPS = ("127.0.0.1", "::1")
+
+LOG_LEVEL = os.environ.get("LOG_LEVEL", "DEBUG" if DEBUG else "INFO")
 
 # Enable custom logging. Please see the Django documentation for detailed guidance on configuring custom logs:
 #   https://docs.djangoproject.com/en/1.11/topics/logging/
 LOGGING = {
     "version": 1,
     "disable_existing_loggers": False,
-    "formatters": {"rq_console": {"format": "%(asctime)s %(message)s", "datefmt": "%H:%M:%S",},},
-    "handlers": {
-        "rq_console": {
-            "level": "DEBUG",
-            "class": "rq.utils.ColorizingStreamHandler",
-            "formatter": "rq_console",
-            "exclude": ["%(asctime)s"],
+    "formatters": {
+        "verbose": {
+            "format": "{asctime} {levelname} {message} - {name} - {module} - {pathname}:{lineno}",
+            "datefmt": "%H:%M:%S",
+            "style": "{",
         },
     },
-    "loggers": {"rq.worker": {"handlers": ["rq_console"], "level": "DEBUG"},},
+    "handlers": {"console": {"level": "DEBUG", "class": "rq.utils.ColorizingStreamHandler", "formatter": "verbose"}},
+    "root": {"handlers": ["console"], "level": LOG_LEVEL},
 }
 
 # Setting this to True will permit only authenticated users to access any part of NetBox. By default, anonymous users
 # are permitted to access most data in NetBox (excluding secrets) but not make any changes.
-LOGIN_REQUIRED = False
-
-# Base URL path if accessing NetBox within a directory. For example, if installed at http://example.com/netbox/, set:
-# BASE_PATH = 'netbox/'
-BASE_PATH = os.environ.get("BASE_PATH", "")
+LOGIN_REQUIRED = is_truthy(os.environ.get("LOGIN_REQUIRED", False))
 
 # Setting this to True will display a "maintenance mode" banner at the top of every page.
-MAINTENANCE_MODE = os.environ.get("MAINTENANCE_MODE", False)
+MAINTENANCE_MODE = is_truthy(os.environ.get("MAINTENANCE_MODE", False))
 
 # An API consumer can request an arbitrary number of objects =by appending the "limit" parameter to the URL (e.g.
 # "?limit=1000"). This setting defines the maximum limit. Setting it to 0 or None will allow an API consumer to request
@@ -149,11 +208,14 @@
 # the default value of this setting is derived from the installed location.
 MEDIA_ROOT = os.environ.get("MEDIA_ROOT", os.path.join(BASE_DIR, "media"))
 
+# Expose Prometheus monitoring metrics at the HTTP endpoint '/metrics'
+METRICS_ENABLED = True
+
 NAPALM_USERNAME = os.environ.get("NAPALM_USERNAME", "")
 NAPALM_PASSWORD = os.environ.get("NAPALM_PASSWORD", "")
 
 # NAPALM timeout (in seconds). (Default: 30)
-NAPALM_TIMEOUT = os.environ.get("NAPALM_TIMEOUT", 30)
+NAPALM_TIMEOUT = int(os.environ.get("NAPALM_TIMEOUT", 30))
 
 # NAPALM optional arguments (see http://napalm.readthedocs.io/en/latest/support/#optional-arguments). Arguments must
 # be provided as a dictionary.
@@ -163,27 +225,34 @@
 }
 
 # Determine how many objects to display per page within a list. (Default: 50)
-PAGINATE_COUNT = os.environ.get("PAGINATE_COUNT", 50)
+PAGINATE_COUNT = int(os.environ.get("PAGINATE_COUNT", 50))
 
 # Enable installed plugins. Add the name of each plugin to the list.
 PLUGINS = ["netbox_onboarding"]
 
-PLUGINS_CONFIG = {"netbox_onboarding": {}}
 # Plugins configuration settings. These settings are used by various plugins that the user may have installed.
 # Each key in the dictionary is the name of an installed plugin and its value is a dictionary of settings.
-# PLUGINS_CONFIG = {}
+PLUGINS_CONFIG = {}
 
 # When determining the primary IP address for a device, IPv6 is preferred over IPv4 by default. Set this to True to
 # prefer IPv4 instead.
-PREFER_IPV4 = os.environ.get("PREFER_IPV4", False)
+PREFER_IPV4 = is_truthy(os.environ.get("PREFER_IPV4", False))
 
 # Remote authentication support
 REMOTE_AUTH_ENABLED = False
-REMOTE_AUTH_BACKEND = "utilities.auth_backends.RemoteUserBackend"
 REMOTE_AUTH_HEADER = "HTTP_REMOTE_USER"
 REMOTE_AUTH_AUTO_CREATE_USER = True
 REMOTE_AUTH_DEFAULT_GROUPS = []
-REMOTE_AUTH_DEFAULT_PERMISSIONS = []
+
+if NETBOX_RELEASE_28 < NETBOX_RELEASE_CURRENT < NETBOX_RELEASE_29:
+    # NetBox 2.8.x Specific Settings
+    REMOTE_AUTH_BACKEND = "utilities.auth_backends.RemoteUserBackend"
+    REMOTE_AUTH_DEFAULT_PERMISSIONS = []
+elif NETBOX_RELEASE_CURRENT < NETBOX_RELEASE_211:
+    REMOTE_AUTH_BACKEND = "netbox.authentication.RemoteUserBackend"
+    REMOTE_AUTH_DEFAULT_PERMISSIONS = {}
+else:
+    raise ImproperlyConfigured(f"Version {NETBOX_RELEASE_CURRENT} of NetBox is unsupported at this time.")
 
 # This determines how often the GitHub API is called to check the latest release of NetBox. Must be at least 1 hour.
 RELEASE_CHECK_TIMEOUT = 24 * 3600
@@ -199,6 +268,10 @@
 # this setting is derived from the installed location.
 REPORTS_ROOT = os.environ.get("REPORTS_ROOT", os.path.join(BASE_DIR, "reports"))
 
+# The file path where custom scripts will be stored. A trailing slash is not needed. Note that the default value of
+# this setting is derived from the installed location.
+SCRIPTS_ROOT = os.environ.get("SCRIPTS_ROOT", os.path.join(BASE_DIR, "scripts"))
+
 # Time zone (default: UTC)
 TIME_ZONE = os.environ.get("TIME_ZONE", "UTC")