Merge pull request #1316 from garberg/py3.6-fixes

Move to Python 3

.github/workflows/ci.yml

@@ -6,14 +6,19 @@ on:
   pull_request:
     branches: [ master ]
 
+env:
+  ACTIONS_RUNNER_DEBUG: true
+  ACTIONS_STEP_DEBUG: true
+
 jobs:
   test:
 
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-20.04
     strategy:
       matrix:
         install: [ pip, apt ]
-        upgrade: [ true, false ]
+        # Upgrade tests temporary disabled as Python 2 version cannot be installed on Ubuntu 20.04
+        upgrade: [ false ]
         exclude:
           - install: pip
             upgrade: true
@@ -26,7 +31,7 @@ jobs:
     - name: "Install dependencies and prepare NIPAP"
       run: |
         # Set up NIPAP repo
-        echo "deb http://spritelink.github.io/NIPAP/repos/apt stable main extra" | sudo tee /etc/apt/sources.list.d/nipap.list
+        echo "deb http://spritelink.github.io/NIPAP/repos/apt testing main extra" | sudo tee /etc/apt/sources.list.d/nipap.list
         wget -O - https://spritelink.github.io/NIPAP/nipap.gpg.key | sudo apt-key add -
         sudo apt update -qq
 
@@ -37,25 +42,24 @@ jobs:
           debhelper \
           dh-python \
           junit4 \
-          python-docutils \
-          python-nose \
-          python-requests \
+          libldap-dev \
+          libsasl2-dev \
+          python3-docutils \
+          python3-nose \
           python3-requests \
-          python-setuptools \
           python3-setuptools \
-          python-wheel \
           python3-wheel \
           python3-all \
           default-jdk \
           gradle \
           rename \
-          postgresql-10 \
-          postgresql-10-ip4r
+          postgresql-12 \
+          postgresql-12-ip4r
 
         # Drop and re-create cluster to be bound to default port
         for version in `pg_lsclusters | tail -n-2 | awk '{ print $1 }'`; do sudo pg_dropcluster $version main; done
-        sudo pg_createcluster --start 10 main
-        sed -e 's/username = guest/username = unittest/' -e 's/password = guest/password = gottatest/' -e 's/port = 1337/port = 1338/' -e "s/#use_ssl = false/use_ssl = true/" nipap-cli/nipaprc > ~/.nipaprc
+        sudo pg_createcluster --start 12 main
+        sed -e 's/username = guest/username = unittest/' -e 's/password = guest/password = gottatest/' nipap-cli/nipaprc > ~/.nipaprc
         chmod 0600 ~/.nipaprc
 
         # Set up CA and generate SSL cert
@@ -74,7 +78,7 @@ jobs:
       if: ${{ matrix.install == 'pip' }}
       run: |
         # install nipap dependencies
-        sudo -H pip install -r nipap/requirements.txt
+        sudo -H pip3 install -r nipap/requirements.txt
         # SQL
         sudo su -c "cd nipap/sql; PGPASSWORD=papin make install" postgres
         # move configuration file into place
@@ -91,7 +95,7 @@ jobs:
         # install nipap-cli dependencies
         sudo -H pip3 install -r nipap-cli/requirements.txt
         # start nipap backend
-        nipap/nipapd --no-pid-file -c /etc/nipap/nipap.conf
+        nipap/nipapd --no-pid-file -c /etc/nipap/nipap.conf -df 2>&1 > /tmp/nipap.log &
 
     - name: "Install latest release from apt"
       if: ${{ matrix.install == 'apt' && matrix.upgrade == true }}
@@ -133,26 +137,33 @@ jobs:
         if [ `grep -c ssl_port /etc/nipap/nipap.conf` -eq 0 ]; then \
           # No SSL config in file - add from scratch
           sudo sed '/^port *=.*/a ssl_port = 1338\nssl_cert_file = \/tmp\/ca\/test.bundle.crt\nssl_key_file = \/tmp\/ca\/test.key' -i /etc/nipap/nipap.conf; \
-          else \
+        else \
           sudo sed -e "s/#ssl_port.\+$/ssl_port = 1338/" -e "s/#ssl_cert_file.\+$/ssl_cert_file = \/tmp\/ca\/test.bundle.crt/" -e "s/#ssl_key_file.\+$/ssl_key_file = \/tmp\/ca\/test.key/" -i /etc/nipap/nipap.conf; \
-          fi
+        fi
         # create local user for unittests
         sudo nipap/nipap-passwd add -u unittest -p gottatest -f /etc/nipap/local_auth.db -n "User for running unit tests"
         sudo nipap/nipap-passwd add -u readonly -p gottatest -f /etc/nipap/local_auth.db --readonly -n "Read-only user for running unit tests"
-        sudo sed -e "s/db_host *= *[^ ]\+/db_host = localhost/" -i /etc/nipap/nipap.conf
-        sudo /etc/init.d/nipapd restart
+        sudo sed -e "s/^db_host *=.*/db_host = localhost/" -e "s/{{SYSLOG}}/true/" -e "s/^debug.\+/debug = true/" -e "s/^user/#user/" -i /etc/nipap/nipap.conf
+        sudo systemctl restart nipapd.service
 
     - name: "Verify pre-upgrade data"
       if: ${{ matrix.upgrade == true }}
-      run: nosetests tests/upgrade-after.py
+      run: nosetests3 tests/upgrade-after.py
 
     - name: "Run test suite"
       env:
         REQUESTS_CA_BUNDLE: /etc/ssl/certs/ca-certificates.crt
       run: |
-        nosetests tests/xmlrpc.py
-        nosetests tests/nipaptest.py
-        nosetests tests/test_cli.py
-        nosetests tests/nipap-ro.py
-        nosetests tests/test_rest.py
+        nosetests3 tests/test_xmlrpc.py
+        nosetests3 tests/nipaptest.py
+        nosetests3 tests/test_cli.py
+        nosetests3 tests/test_nipap_ro.py
+        nosetests3 tests/test_rest.py
         make -C jnipap test
+
+    - name: "Accident analysis"
+      if: failure()
+      run: |
+        sudo cat /etc/nipap/nipap.conf
+        sudo cat /var/log/syslog
+        sudo cat /tmp/nipap.log || true

Dockerfile.nipapd

@@ -32,7 +32,7 @@
 # via a volume.
 #
 
-FROM ubuntu:xenial
+FROM ubuntu:bionic
 MAINTAINER Kristian Larsson <[email protected]>
 
 ENV DEBIAN_FRONTEND=noninteractive
@@ -44,19 +44,30 @@ RUN apt-get update -qy && apt-get upgrade -qy \
     libpq-dev \
     libsqlite3-dev \
     postgresql-client \
-    python \
-    python-all \
-    python-docutils \
-    python-pip \
-    python-dev \
- && pip --no-input install envtpl \
+    software-properties-common \
+    python3 \
+    python3-all \
+    python3-pip \
+    python3-dev \
+    libsasl2-dev \
+    libldap2-dev \
+    libssl-dev \
  && rm -rf /var/lib/apt/lists/*
 
+# Install any additional CA certs from ca_certs folder required by corp proxies etc
+RUN mkdir -p /usr/share/ca-certificates/extra
+COPY ca_certs/*.crt /usr/share/ca-certificates/extra/
+RUN ls /usr/share/ca-certificates/extra/*.crt | sed 's/\/usr\/share\/ca-certificates\///g' >> /etc/ca-certificates.conf
+RUN update-ca-certificates
+RUN pip3 install --upgrade pip
+RUN pip3 config set global.cert /etc/ssl/certs/ca-certificates.crt
+
 
 COPY nipap /nipap
 WORKDIR /nipap
-RUN pip --no-input install -r requirements.txt \
- && python setup.py install
+RUN pip3 install --no-input envtpl
+RUN pip3 --no-input install -r requirements.txt \
+ && python3 setup.py install
 
 EXPOSE 1337
 ENV LISTEN_ADDRESS=0.0.0.0 LISTEN_PORT=1337 SYSLOG=false DB_PORT=5432 DB_SSLMODE=disable

ca_certs/README.rst

@@ -0,0 +1,8 @@
+Custom CA Certs for docker containers
+=====================================
+If you need to include specific CA certs which you must trust, place them here
+in PEM format, named \*.crt.
+
+This may be required if you need to build the container from inside a network
+which uses a proxy or similar, or other dependencies towards internal services
+are included in your containers.

docs/config-www.rst

@@ -53,7 +53,7 @@ Apache httpd with mod_wsgi
 ==========================
 Begin by installing Apache httpd with mod_wsgi::
 
-    apt-get install libapache2-mod-wsgi
+    apt-get install libapache2-mod-wsgi-py3
 
 Then, add a new virtual host or configure the default one with the line::
 

nipap-cli/Makefile

@@ -38,7 +38,7 @@ test:
 check:
 
 upload:
-	python setup.py sdist upload
+	$(PYTHON) setup.py sdist upload
 
 clean:
 	$(PYTHON) setup.py clean

nipap-www/Makefile

@@ -1,7 +1,7 @@
 # $Id: Makefile,v 1.6 2011/04/18 17:14:00 lukagarb Exp $
 #
 
-PYTHON=`which python`
+PYTHON=`which python3`
 DESTDIR=/
 BUILDIR=$(CURDIR)/debian/nipap-www-build
 PROJECT=nipap-www
@@ -27,7 +27,7 @@ buildrpm:
 builddeb:
 	# build the source package in the parent directory
 	# then rename it to project_version.orig.tar.gz
-	$(PYTHON) setup.py sdist --dist-dir=../ # --prune
+	$(PYTHON) setup.py sdist --dist-dir=../ #--prune
 	rename -f 's/$(PROJECT)-(\d.*)\.tar\.gz/$(PROJECT)_$$1\.orig\.tar\.gz/' ../*
 	# build the package
 	debuild -us -uc -sa
@@ -38,7 +38,7 @@ test:
 check:
 
 upload:
-	python setup.py sdist upload
+	$(PYTHON) setup.py sdist upload
 
 clean:
 	$(PYTHON) setup.py clean

nipap-www/debian/control

@@ -2,13 +2,12 @@ Source: nipap-www
 Section: web
 Priority: optional
 Maintainer: Lukas Garberg <[email protected]>
-Build-Depends: debhelper (>= 5.0.38), debhelper (>= 7),
- python (>= 2.7), python-setuptools (>= 0.6b3)
+Build-Depends: debhelper (>= 5.0.38), debhelper (>= 7), python3 (>= 3.6), python3-setuptools (>= 0.6b3), dh-python
 Standards-Version: 4.4.0
 
 Package: nipap-www
 Architecture: all
-Depends: debconf, python (>= 2.7), ${misc:Depends}, python-flask, python-pynipap, nipap-common, python-jinja2
+Depends: debconf, python3 (>= 3.6), ${misc:Depends}, python3-flask, python3-pynipap, nipap-common, python3-jinja2
 XB-Python-Version: ${python:Versions}
 Description: web frontend for NIPAP
  A web UI for the NIPAP IP address planning service.

nipap-www/debian/pydist-overrides

@@ -1,3 +1,3 @@
-Jinja2 python-jinja2
-pynipap python-pynipap
+Jinja2 python3-jinja2
+pynipap python3-pynipap
 nipap nipap-common

nipap-www/debian/rules

@@ -1,6 +1,6 @@
 #!/usr/bin/make -f
 
 %:
-	dh $@ --with python2 --buildsystem=python_distutils
+	dh $@ --with python3 --buildsystem=python_distutils
 
 

nipap-www/nipapwww/__init__.py

@@ -1,11 +1,3 @@
-import os
-
-from flask import Flask, redirect, url_for
-
-from nipap.nipapconfig import NipapConfig
-
-import pynipap
-
 __version__     = "0.31.2"
 __author__      = "Kristian Larsson, Lukas Garberg"
 __author_email__ = "[email protected], [email protected]"
@@ -15,6 +7,14 @@
 
 
 def create_app(test_config=None):
+
+    # Moved imports here to be able to import this module without having the
+    # dependencies installed. Relevant during initial package build.
+    import os
+    from flask import Flask, redirect, url_for
+    from nipap.nipapconfig import NipapConfig
+    import pynipap
+
     # create and configure the app
     app = Flask(__name__, instance_relative_config=True)
     app.config.from_mapping(

nipap-www/nipapwww/ng.py

@@ -9,7 +9,7 @@
 
 from flask import Blueprint, redirect, render_template, url_for
 
-from auth import login_required
+from .auth import login_required
 
 bp = Blueprint('ng', __name__, url_prefix='/ng')
 

nipap-www/nipapwww/prefix.py

@@ -1,6 +1,6 @@
 from flask import Blueprint, g, render_template
 
-from auth import login_required
+from .auth import login_required
 
 bp = Blueprint('prefix', __name__, url_prefix='/prefix')
 

nipap-www/nipapwww/xhr.py

@@ -4,15 +4,15 @@
 
 from pynipap import NipapError, Pool, Prefix, Tag, VRF
 
-from auth import login_required
+from .auth import login_required
 
 
 bp = Blueprint('xhr', __name__, url_prefix='/xhr')
 
 
 def validate_string(req, key):
 
-    if isinstance(req[key], basestring) and req[key].strip() != '':
+    if isinstance(req[key], str) and req[key].strip() != '':
         return req[key].strip()
     else:
         return None
@@ -591,7 +591,7 @@ def add_prefix():
     # Sanitize input parameters
     if 'vrf' in request.json:
         try:
-            if request.json['vrf'] is None or len(unicode(request.json['vrf'])) == 0:
+            if request.json['vrf'] is None or len(str(request.json['vrf'])) == 0:
                 p.vrf = None
             else:
                 p.vrf = VRF.get(int(request.json['vrf']))
@@ -724,7 +724,7 @@ def edit_prefix(id):
         if 'vrf' in request.json:
 
             try:
-                if request.json['vrf'] is None or len(unicode(request.json['vrf'])) == 0:
+                if request.json['vrf'] is None or len(str(request.json['vrf'])) == 0:
                     p.vrf = None
                 else:
                     p.vrf = VRF.get(int(request.json['vrf']))
@@ -802,7 +802,7 @@ def del_current_vrf():
     """ Remove VRF to filter list session variable
     """
 
-    vrf_id = unicode(request.json['vrf_id'])
+    vrf_id = str(request.json['vrf_id'])
     if vrf_id in session['current_vrfs']:
         del session['current_vrfs'][vrf_id]
         session.modified = True
@@ -820,7 +820,7 @@ def get_current_vrfs():
     """
 
     # Verify that all currently selected VRFs still exists
-    cur_vrfs = session.get('current_vrfs', {}).items()
+    cur_vrfs = list(session.get('current_vrfs', {}).items())
     if len(cur_vrfs) > 0:
         q = {
             'operator': 'equals',

nipap/MANIFEST.in

@@ -1 +1,3 @@
 include README.rst MANIFEST.in
+include *.man.rst
+include requirements.txt

nipap/Makefile

@@ -1,7 +1,7 @@
 # $Id: Makefile,v 1.6 2011/04/18 17:14:00 lukagarb Exp $
 #
 
-PYTHON=`which python2`
+PYTHON=`which python3`
 DESTDIR=/
 BUILDIR=$(CURDIR)/debian/python-nipap-build
 PROJECT=nipap
@@ -27,7 +27,7 @@ buildrpm: sql2py
 builddeb: sql2py
 	# build the source package in the parent directory
 	# then rename it to project_version.orig.tar.gz
-	$(PYTHON) setup.py sdist --dist-dir=../ --prune
+	$(PYTHON) setup.py sdist --dist-dir=../ #--prune
 	rename -f 's/$(PROJECT)-(\d.*)\.tar\.gz/$(PROJECT)_$$1\.orig\.tar\.gz/' ../*
 	# build the package
 	debuild -us -uc
@@ -38,7 +38,7 @@ test:
 check:
 
 upload:
-	python setup.py sdist upload
+	$(PYTHON) setup.py sdist upload
 
 clean:
 	$(PYTHON) setup.py clean