Merge pull request #427 from lneuhaus/develop-0.9.3

Development work from 2017-2020 ("develop-0.9.3")

.deploy_to_sourceforge.py

@@ -7,14 +7,15 @@
     if len(sys.argv) < 2:
         print("Usage: python .deploy_to_sourceforge.py file1 [file2] ...")
 
-    pw = os.environ['PYPI_PASSWORD']
+    pw = os.environ['PYPI_PSW']
     ssh = sshshell.SshShell(hostname='frs.sourceforge.net',
                             user='lneuhaus',
                             password=pw,
                             shell=False)
     for filename in sys.argv[1:]:
-        for destpath in ['/home/frs/project/pyrpl/%s/' % __version__,
-                        '/home/frs/project/pyrpl/']:
+        for destpath in ['/home/frs/project/pyrpl/',
+                         '/home/frs/project/pyrpl/%s/' % __version__
+                         ]:
             print("Uploading file '%s' to '%s' on sourceforge..." % (filename, destpath))
             try:
                 ssh.scp.put(filename, destpath)

.docker/all_python_build.sh

@@ -0,0 +1,7 @@
+docker system prune -a
+docker build --build-arg PYTHON_VERSION=3.7 -t python-37 ../.
+docker build --build-arg PYTHON_VERSION=3.6 -t python-36 ../.
+docker build --build-arg PYTHON_VERSION=3.5 -t python-35 ../.
+docker build --build-arg PYTHON_VERSION=2.7 -t python-27 ../.
+docker build --build-arg PYTHON_VERSION=3 -t python-3 ../.
+docker build -t python ../.

.docker/build.sh

@@ -0,0 +1 @@
+docker build -t pyrpl ../.

.docker/run-27.sh

@@ -0,0 +1,7 @@
+docker run -ti --rm \
+       -e DISPLAY \
+       -v /tmp/.X11-unix:/tmp/.X11-unix \
+       -v /home/leo/github/pyrpl:/home/pyrpl \
+       -v /home/leo/github/pyrpl-copy:/home/pyrpl-copy \
+       --net=host \
+       python-27

.docker/run-3.sh

@@ -0,0 +1,7 @@
+docker run -ti --rm \
+       -e DISPLAY \
+       -v /tmp/.X11-unix:/tmp/.X11-unix \
+       -v /home/leo/github/pyrpl:/home/pyrpl \
+       -v /home/leo/github/pyrpl-copy:/home/pyrpl-copy \
+       --net=host \
+       python-3

.docker/run-35.sh

@@ -0,0 +1,7 @@
+docker run -ti --rm \
+       -e DISPLAY \
+       -v /tmp/.X11-unix:/tmp/.X11-unix \
+       -v /home/leo/github/pyrpl:/home/pyrpl \
+       -v /home/leo/github/pyrpl-copy:/home/pyrpl-copy \
+       --net=host \
+       python-35

.docker/run-36.sh

@@ -0,0 +1,7 @@
+docker run -ti --rm \
+       -e DISPLAY \
+       -v /tmp/.X11-unix:/tmp/.X11-unix \
+       -v /home/leo/github/pyrpl:/home/pyrpl \
+       -v /home/leo/github/pyrpl-copy:/home/pyrpl-copy \
+       --net=host \
+       python-36

.docker/run.sh

@@ -0,0 +1,7 @@
+docker run -ti --rm \
+       -e DISPLAY \
+       -v /tmp/.X11-unix:/tmp/.X11-unix \
+       -v /home/leo/github/pyrpl:/home/pyrpl \
+       -v /home/leo/github/pyrpl-copy:/home/pyrpl-copy \
+       --net=host \
+       python-37

.docker/setup_jenkins.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+echo 'JAVA_ARGS="-Dhudson.tasks.MailSender.SEND_TO_UNKNOWN_USERS=true"' >> /etc/default/jenkins
+xhost +local:docker
+

.docker/test/Dockerfile

@@ -0,0 +1,56 @@
+# define base image
+FROM ubuntu:latest
+# FROM node:7-onbuild
+
+# set maintainer
+LABEL maintainer "[email protected]"
+
+USER root
+
+ARG CONDA_DIR="/opt/conda"
+ARG PYTHON_VERSION="3"
+
+# setup ubuntu with gui support
+RUN apt update --yes
+RUN apt upgrade --yes
+RUN apt update --yes
+RUN apt-get install --yes systemd wget sloccount qt5-default
+# sets up keyboard support in GUI
+ENV QT_XKB_CONFIG_ROOT /usr/share/X11/xkb
+
+# install miniconda
+RUN mkdir /tmp/miniconda
+WORKDIR /tmp/miniconda
+RUN if [ "$PYTHON_VERSION" = "2" ] ; then wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O Miniconda.sh; else wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O Miniconda.sh; fi
+RUN chmod +x Miniconda.sh
+RUN ./Miniconda.sh -b -p $CONDA_DIR
+
+# set path environment variable to refer to conda bin dir (we are working in the (base) conda environment
+ENV PATH="$CONDA_DIR/bin:$PATH"
+ENV LD_LIBRARY_PATH="$CONDA_DIR/lib:$LD_LIBRARY_PATH"
+
+RUN python -V
+
+# install desired python version and additional packages
+RUN conda install --yes python=$PYTHON_VERSION numpy scipy paramiko pandas jupyter nose pip pyqt qtpy nbconvert coverage twine matplotlib
+
+RUN python -V
+
+RUN pip install radon
+RUN conda install --yes  nb_conda nb_conda_kernels nb_anacondacloud
+
+RUN python -V
+
+# Clean up miniconda installation files
+WORKDIR /
+RUN rm -rf /tmp/miniconda
+
+# auxiliary environment variable
+ENV PYTHON_VERSION=$PYTHON_VERSION
+
+# print a message
+RUN echo "Docker image is up and running...."
+RUN echo $PATH
+
+# print some python diagnostics information
+RUN python -V

.gitignore

@@ -43,6 +43,11 @@ pyrpl/fpga/red_pitaya.prm
 pyrpl/fpga/*.dmp
 pyrpl/.idea
 
+# Vivado project-related stuff
+pyrpl/fpga/project/.*
+pyrpl/fpga/project/
+*.jou
+
 #fpga-related
 fpga/out/
 fpga/sdk/
@@ -63,12 +68,14 @@ htmlcov/
 .tox/
 .coverage
 .coverage.*
+cover/*
 .cache
 nosetests.xml
 coverage.xml
 *,cover
 .hypothesis/
 xunit.xml
+unit_tests.xml
 
 # Translations
 *.mo
@@ -123,3 +130,4 @@ scripts/build/
 scripts/__init__.py
 scripts/run_pyrpl.spec
 scripts/pyrpl.spec
+cover/index.html

.travis.yml

@@ -4,29 +4,36 @@
 
 # lots of stuff here comes from https://gist.github.com/dan-blanchard/7045057
 
-env:
-- AUTOCORRETPEP8=0 DISPLAY=:99.0
 # REDPITAYA variables are defined in travis account (encrypted)
 
-language: python
-# nosetests is only executed for 2.7, 3.5 and 3.6, but we still test the
-# installation for other python versions (3.4)
-python:
-  - "2.7"
-  - "3.6"
-  - "3.5"
-  - "3.4"
-
 notifications:
-  email: false
+  email:
+    recipients:
+      - [email protected]
+
+language: generic
+
+env:
+  - TRAVIS_PYTHON_VERSION=3.7
+os:
+  - osx
+  - linux
 
 before_install:
-  - sudo apt-get update
+  - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
+    sudo apt-get update;
+    fi
   # We do this conditionally because it saves us some downloading.
-  - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then
+  - if [[ "$TRAVIS_OS_NAME" == "linux" && "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then
     wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O miniconda.sh;
-    else
+    elif [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
     wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh;
+    elif [[ "$TRAVIS_OS_NAME" == "osx" && "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then
+    wget https://repo.continuum.io/miniconda/Miniconda2-latest-MacOSX-x86_64.sh -O miniconda.sh;
+    elif [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
+    wget https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -O miniconda.sh;
+    else
+    echo "Invalid combination of OS ($TRAVIS_OS_NAME) and Python version ($TRAVIS_PYTHON_VERSION)";
     fi
   - chmod +x miniconda.sh
   - bash miniconda.sh -b -p $HOME/miniconda
@@ -37,56 +44,62 @@ before_install:
   # Useful for debugging any issues with conda
   - conda info -a
   # The next lines fix a crash with multiprocessing on Travis and are not specific to using Miniconda
-  - sudo rm -rf /dev/shm
-  - sudo ln -s /run/shm /dev/shm
+  - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
+    sudo rm -rf /dev/shm; sudo ln -s /run/shm /dev/shm;
+    fi
   # starts gui support, see https://docs.travis-ci.com/user/gui-and-headless-browsers/
-  - sh -e /etc/init.d/xvfb start
+  # and https://github.com/travis-ci/travis-ci/issues/7313#issuecomment-279914149 (for MacOSX)
+  # formerly "sh -e sudo Xvfb :99 -ac -screen 0 1024x768x8";
+  - export DISPLAY=":99.0"
+  - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
+    sh -e /etc/init.d/xvfb start;
+    fi
+  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
+    Xvfb :98 -ac -screen 0 1024x768x8;
+    fi &
   # give it some time to start
   - sleep 3
 
 install:
   # avoid to get cancelled because of very long tests
   # we get issues with building numpy etc if we do not include those in the next line
-  - conda create -q -n test-environment python=$TRAVIS_PYTHON_VERSION numpy scipy paramiko pandas nose pip pyqt qtpy
+  - conda create -q -n test-environment python=$TRAVIS_PYTHON_VERSION numpy scipy paramiko pandas nose pip pyqt qtpy nbconvert
   - source activate test-environment
-  - if [[ "$TRAVIS_PYTHON_VERSION" != "3.3" ]]; then
-    conda install --yes -c conda-forge coveralls;
-    fi
-  # overwrite default global config file with a custom one for travis (allows slower communication time)
-  - \cp ./travis_global_config.yml ./pyrpl/config/global_config.yml
+  # convert readme file to rst for PyPI
+  - conda install pandoc
+  - pandoc --from=markdown --to=rst --output=README.rst README.md
+  # install pyinstaller
+  - cd ..
+  - git clone https://www.github.com/lneuhaus/pyinstaller.git -b develop --depth=1
+  - cd pyinstaller
+  - git status
+  - python setup.py develop
+  - cd ..
+  - cd pyrpl
+  # install pyrpl
   - python setup.py install
-  # packages for coverage reports
-  - pip install coverage codecov
 
-# Run test
+# create, test and upload binary for mac os
 script:
-  - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]] || [[ "$TRAVIS_PYTHON_VERSION" == "3.4" ]] || [[ "$TRAVIS_PYTHON_VERSION" == "3.6" ]]; then
-    nosetests;
-    fi
+  - export QT_QPA_PLATFORM_PLUGIN_PATH=$HOME/miniconda/bin/envs/test-environment/Library/plugins/platforms
+  - pyinstaller pyrpl.spec
+  - mv dist/pyrpl ./pyrpl-mac-develop
+  - chmod 755 pyrpl-mac-develop
+  - (./pyrpl-mac-develop config=test_osx hostname=_FAKE_ &)
+  - PYRPL_PID=$!
+  - sleep 30
+  - killall pyrpl-mac-develop
+  - python .deploy_to_sourceforge.py pyrpl-mac-develop
 
-after_script:
-  - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]] || [[ "$TRAVIS_PYTHON_VERSION" == "3.4" ]] || [[ "$TRAVIS_PYTHON_VERSION" == "3.6" ]]; then
-    codecov;
-    fi
-
-# automatic release when a new tag is created needs a few preliminary steps...
-# create a Readme.rst for PyPI
-# make an executable for linux and upload to sourceforge
+# automatic release when a new tag is created: before_deploy, deploy, and after_deploy
 before_deploy:
-  - if [[ "$TRAVIS_PYTHON_VERSION" == "3.4" ]]; then
-    source activate test-environment;
-    conda install pandoc;
-    pip install pyinstaller;
-    pandoc --from=markdown --to=rst --output=README.rst README.md;
-    pyinstaller pyrpl.spec;
-    mv dist/pyrpl ./pyrpl-linux;
-    python .deploy_to_sourceforge.py pyrpl-linux;
-    fi
+  - echo Deploy
+  - source activate test-environment
 
 deploy:
   provider: pypi
   user: lneuhaus
-  password: $PYPI_PASSWORD
+  password: $PYPI_PSW
   skip_cleanup: true
   on:
     tags: true

CHANGELOG.md

@@ -0,0 +1,9 @@
+# PyRPL changelog
+
+
+## 0.9.5 (November 17, 2020)
+
+- merges the "0.9.3-develop" branch with accumulated upgrades from over 2 years
+- last version to support Python 2.7 (though not running tests any more)
+- tested on Python 3.6 and 3.7
+- significant improvements to IIR filter module

Dockerfile

@@ -0,0 +1,48 @@
+# define base image
+FROM ubuntu:latest
+# FROM node:7-onbuild
+
+# set maintainer
+LABEL maintainer "[email protected]"
+
+USER root
+
+ARG CONDA_DIR="/opt/conda"
+ARG PYTHON_VERSION="3"
+
+# setup ubuntu with gui support
+RUN apt update --yes
+RUN apt upgrade --yes
+RUN apt update --yes
+RUN apt-get install --yes systemd wget sloccount qt5-default binutils
+# sets up keyboard support in GUI
+ENV QT_XKB_CONFIG_ROOT /usr/share/X11/xkb
+
+# install miniconda
+RUN mkdir /tmp/miniconda
+WORKDIR /tmp/miniconda
+RUN if [ "$PYTHON_VERSION" = "2" ] ; then wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O Miniconda.sh; else wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O Miniconda.sh; fi
+RUN chmod +x Miniconda.sh
+RUN ./Miniconda.sh -b -p $CONDA_DIR
+
+# set path environment variable to refer to conda bin dir (we are working in the (base) conda environment
+ENV PATH="$CONDA_DIR/bin:$PATH"
+# set library path until pyinstaller issue is fixed
+ENV LD_LIBRARY_PATH="$CONDA_DIR/lib:$LD_LIBRARY_PATH"
+
+# install desired python version and additional packages
+RUN conda install --yes python=$PYTHON_VERSION numpy scipy paramiko pandas jupyter nose pip pyqt qtpy nbconvert coverage twine matplotlib nb_conda_kernels
+
+# Clean up miniconda installation files
+WORKDIR /
+RUN rm -rf /tmp/miniconda
+
+# auxiliary environment variable
+ENV PYTHON_VERSION=$PYTHON_VERSION
+
+# print a message
+RUN echo "Docker image is up and running...."
+RUN echo $PATH
+
+# print some python diagnostics information
+RUN python -V