Include NOTICE.txt in release

With this commit we generate a NOTICE.txt file and include it in the
Rally wheel as well as the offline distribution package.

Relates opensearch-project#622

.gitignore

@@ -107,6 +107,8 @@ target/
 .rally_it/
 # Virtualenv for development
 .venv*/
+# Autogenerated files
+NOTICE.txt
 
 # Emacs backup files
 *~

Makefile

@@ -68,7 +68,7 @@ install: venv-create
 clean: nondocs-clean docs-clean
 
 nondocs-clean:
-	rm -rf .benchmarks .eggs .tox .rally_it .cache build dist esrally.egg-info logs junit-py*.xml
+	rm -rf .benchmarks .eggs .tox .rally_it .cache build dist esrally.egg-info logs junit-py*.xml NOTICE.txt
 
 docs-clean:
 	cd docs && $(MAKE) clean

create-notice.sh

@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#	http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# fail this script immediately if any command fails with a non-zero exit code
+set -e
+# Treat unset env variables as an error
+set -u
+# fail on pipeline errors, e.g. when grepping
+set -o pipefail
+
+readonly OUTPUT_FILE="${__NOTICE_OUTPUT_FILE}"
+
+
+function add_license {
+    local dep_name=$1
+    local download_url=$2
+
+    printf "\n======================================\n${dep_name} LICENSE\n======================================\n" >> "${OUTPUT_FILE}"
+    curl --fail --show-error --silent "${download_url}" >> "${OUTPUT_FILE}"
+}
+
+function main {
+    cat NOTICE > "${OUTPUT_FILE}"
+    # direct dependencies
+    printf "\n======================================\ncertifi\n======================================\n\n" >> "${OUTPUT_FILE}"
+    # link to a URL providing the MPL-covered source code
+    printf "The source code can be obtained at https://github.com/certifi/python-certifi\n" >> "${OUTPUT_FILE}"
+    add_license "certifi" "https://raw.githubusercontent.com/certifi/python-certifi/master/LICENSE"
+    add_license "elasticsearch" "https://raw.githubusercontent.com/elastic/elasticsearch-py/master/LICENSE"
+    add_license "jinja2" "https://raw.githubusercontent.com/pallets/jinja/master/LICENSE"
+    add_license "jsonschema" "https://raw.githubusercontent.com/Julian/jsonschema/master/COPYING"
+    add_license "psutil" "https://raw.githubusercontent.com/giampaolo/psutil/master/LICENSE"
+    add_license "py-cpuinfo" "https://raw.githubusercontent.com/workhorsy/py-cpuinfo/master/LICENSE"
+    add_license "tabulate" "https://bitbucket.org/astanin/python-tabulate/raw/03182bf9b8a2becbc54d17aa7e3e7dfed072c5f5/LICENSE"
+    add_license "thespian" "https://raw.githubusercontent.com/kquick/Thespian/master/LICENSE.txt"
+
+    # transitive dependencies
+    # Jinja2 -> Markupsafe
+    add_license "Markupsafe" "https://raw.githubusercontent.com/pallets/markupsafe/master/LICENSE.rst"
+    # elasticsearch -> urllib3
+    add_license "urllib3" "https://raw.githubusercontent.com/shazow/urllib3/master/LICENSE.txt"
+}
+
+main

release.sh

@@ -67,6 +67,10 @@ then
     exit 2
 fi
 
+__NOTICE_OUTPUT_FILE="NOTICE.txt"
+echo "Preparing ${__NOTICE_OUTPUT_FILE}"
+source create-notice.sh
+
 # Build new version
 python3 setup.py bdist_wheel
 # Upload to PyPI

scripts/offline-install.sh

@@ -38,6 +38,15 @@ readonly RELATIVE_DOWNLOAD_DIR="esrally-dist-${RALLY_VERSION}"
 readonly ABSOLUTE_DOWNLOAD_DIR="${WD}/${RELATIVE_DOWNLOAD_DIR}"
 readonly ABSOLUTE_DOWNLOAD_BIN_DIR="${ABSOLUTE_DOWNLOAD_DIR}/bin"
 
+# see http://stackoverflow.com/a/246128
+SOURCE="${BASH_SOURCE[0]}"
+while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
+  DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+  SOURCE="$(readlink "$SOURCE")"
+  [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
+done
+SCRIPT_SRC_HOME="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+
 function main {
     local archive_name="esrally-dist-${RALLY_VERSION}.tar.gz"
     local install_script_file="install.sh"
@@ -49,22 +58,10 @@ function main {
     # Prepare install
     pip3 download esrally=="${RALLY_VERSION}" --dest "${ABSOLUTE_DOWNLOAD_BIN_DIR}"
 
-    # grab all license files
-    echo "Downloading license files for all dependencies"
-    # direct dependencies
-    curl --silent -o "${ABSOLUTE_DOWNLOAD_BIN_DIR}/certifi-LICENSE.txt" https://raw.githubusercontent.com/certifi/python-certifi/master/LICENSE
-    curl --silent -o "${ABSOLUTE_DOWNLOAD_BIN_DIR}/elasticsearch-LICENSE.txt" https://raw.githubusercontent.com/elastic/elasticsearch-py/master/LICENSE
-    curl --silent -o "${ABSOLUTE_DOWNLOAD_BIN_DIR}/jinja2-LICENSE.txt" https://raw.githubusercontent.com/pallets/jinja/master/LICENSE
-    curl --silent -o "${ABSOLUTE_DOWNLOAD_BIN_DIR}/jsonschema-LICENSE.txt" https://raw.githubusercontent.com/Julian/jsonschema/master/COPYING
-    curl --silent -o "${ABSOLUTE_DOWNLOAD_BIN_DIR}/psutil-LICENSE.txt" https://raw.githubusercontent.com/giampaolo/psutil/master/LICENSE
-    curl --silent -o "${ABSOLUTE_DOWNLOAD_BIN_DIR}/py-cpuinfo-LICENSE.txt" https://raw.githubusercontent.com/workhorsy/py-cpuinfo/master/LICENSE
-    curl --silent -o "${ABSOLUTE_DOWNLOAD_BIN_DIR}/tabulate-LICENSE.txt" https://bitbucket.org/astanin/python-tabulate/raw/03182bf9b8a2becbc54d17aa7e3e7dfed072c5f5/LICENSE
-    curl --silent -o "${ABSOLUTE_DOWNLOAD_BIN_DIR}/thespian-LICENSE.txt" https://raw.githubusercontent.com/kquick/Thespian/master/LICENSE.txt
-    # transitive dependencies
-    # Jinja2 -> Markupsafe
-    curl --silent -o "${ABSOLUTE_DOWNLOAD_BIN_DIR}/markupsafe-LICENSE.txt" https://raw.githubusercontent.com/pallets/markupsafe/master/LICENSE
-    # elasticsearch -> urllib3
-    curl --silent -o "${ABSOLUTE_DOWNLOAD_BIN_DIR}/urllib3-LICENSE.txt" https://raw.githubusercontent.com/shazow/urllib3/master/LICENSE.txt
+
+    echo "Preparing NOTICE file"
+    __NOTICE_OUTPUT_FILE="${ABSOLUTE_DOWNLOAD_BIN_DIR}/NOTICE.txt"
+    source "${SCRIPT_SRC_HOME}/../create-notice.sh"
 
     # create an offline install script
     cat >"${install_script}" <<EOL

setup.py

@@ -38,23 +38,32 @@ def str_from_file(name):
 
 ################################################################################################
 #
-# Adapt `scripts/offline-install.sh` whenever changing dependencies here.
+# Adapt `create-notice.sh` whenever changing dependencies here.
 #
-# That script grabs all license files so we include them in the offline package.
+# That script grabs all license files so we include them in the notice file.
 #
 ################################################################################################
 install_requires = [
+    # License: Apache 2.0
+    # transitive dependency urllib3: MIT
     "elasticsearch==6.2.0",
+    # License: BSD
     "psutil==5.4.0",
+    # License: MIT
     "py-cpuinfo==3.2.0",
+    # License: MIT
     "tabulate==0.8.1",
+    # License: MIT
     "jsonschema==2.5.1",
+    # License: BSD
+    # transitive dependency Markupsafe: BSD
     "Jinja2==2.9.5",
-    # remote messaging
+    # License: MIT
     "thespian==3.9.3",
     # recommended library for thespian to identify actors more easily with `ps`
     # "setproctitle==1.1.10",
     # always use the latest version, these are certificate files...
+    # License: MPL 2.0
     "certifi"
 ]