Merge remote-tracking branch 'upstream/branch-23.12' into grelee/buil…

…d-using-pyproject-toml

ci/release/update-version.sh

@@ -37,6 +37,7 @@ sed_runner 's/release = .*/release = '"'${NEXT_FULL_TAG}'"'/g' docs/source/conf.
 sed_runner "s/^version = .*/version = \"${NEXT_FULL_TAG}\"/g" python/cucim/pyproject.toml
 sed_runner "s/${CURRENT_LONG_TAG}/${NEXT_FULL_TAG}/g" VERSION
 sed_runner "s/${CURRENT_LONG_TAG}/${NEXT_FULL_TAG}/g" python/cucim/VERSION
+sed_runner "s/__version__ = .*/__version__ = \"${NEXT_FULL_TAG}\"/g" python/cucim/src/cucim/__init__.pyi
 sed_runner "s/${CURRENT_LONG_TAG}/${NEXT_FULL_TAG}/g" cpp/plugins/cucim.kit.cuslide/VERSION
 sed_runner "s/${CURRENT_LONG_TAG}/${NEXT_FULL_TAG}/g" cpp/plugins/cucim.kit.cumed/VERSION
 sed_runner "s#\[Version ${CURRENT_LONG_TAG}\](release_notes/v${CURRENT_LONG_TAG}.md)#\[Version ${NEXT_FULL_TAG}\](release_notes/v${NEXT_FULL_TAG}.md)#g" python/cucim/docs/index.md

python/cucim/pyproject_.toml

@@ -0,0 +1,129 @@
+# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved.
+# See file LICENSE for terms.
+
+[tool.isort]
+profile = "black"
+force_single_line = false
+line_length = 80
+forced_separate = "test_cucim"
+force_grid_wrap = 0
+multi_line_output = 3
+order_by_type = true
+combine_as_imports = true
+include_trailing_comma = true
+known_first_party = [
+    "cucim",
+]
+default_section = "THIRDPARTY"
+sections = [
+    "FUTURE",
+    "STDLIB",
+    "THIRDPARTY",
+    "FIRSTPARTY",
+    "LOCALFOLDER",
+]
+skip = [
+    "3rdparty",
+    "thirdparty",
+    ".eggs",
+    ".git",
+    ".hg",
+    ".mypy_cache",
+    ".tox",
+    ".venv",
+    "ci",
+    "cpp",
+    "_build",
+    "build",
+    "build-debug",
+    "build-release",
+    "_deps",
+    "dist",
+]
+
+[tool.pytest.ini_options]
+# If a pytest section is found in one of the possible config files
+# (pytest.ini, tox.ini or setup.cfg), then pytest will not look for any others,
+# so if you add a pytest config section elsewhere,
+# you will need to delete this section from setup.cfg.
+norecursedirs = "migrations"
+python_files = [
+    "test_*.py",
+    "*_test.py",
+    "tests.py",
+]
+addopts = [
+    "-ra",
+    "--strict-markers",
+    # --doctest-modules",
+    # --doctest-glob=\*.rst",
+    "--tb=short",
+    "--ignore-glob=build*",
+]
+testpaths = [
+    "src",
+    "tests",
+]
+
+[tool.ruff]
+select = ["E", "F", "W"]
+fixable = ["ALL"]
+exclude = [
+    # TODO: Remove this in a follow-up where we fix __all__.
+    ".tox",
+    ".eggs",
+    "ci/templates",
+    "build",
+    "dist",
+    ".git",
+    "__pycache__",
+    "doc/conf.py",
+    "doc/sphinxext",
+    "__init__.py",
+]
+line-length = 80
+
+[tool.ruff.per-file-ignores]
+# "src/cucim/skimage/util/tests/test_shape.py" = ["E201", "E202"]
+
+[tool.black]
+line-length = 80
+target-version = ["py39"]
+include = '\.py?$'
+exclude = '''
+/(
+    3rdparty |
+    thirdparty |
+    \.eggs |
+    \.git |
+    \.hg |
+    \.mypy_cache |
+    \.tox |
+    \.venv |
+    _build |
+    _deps |
+    cpp |
+    ci |
+    build |
+    build-debug |
+    build-release |
+    dist |
+    docker |
+    docs |
+)/
+'''
+
+[tool.setuptools]
+license-files = ["LICENSE"]
+
+[tool.codespell]
+# note: pre-commit passes explicit lists of files here, which this skip file list doesn't override -
+# this is only to allow you to run codespell interactively
+# e.g. via
+#   codespell --toml python/cucim/pyproject.toml . -i 3 -w
+skip = "build*,dist,.cache,html,_build,_deps,3rdparty,_static,generated,latex,.git,*.ipynb,test_data/input/LICENSE-3rdparty,jitify_testing"
+# ignore-regex = ""
+ignore-words-list = "ans,coo,boun,bui,gool,hart,lond,nd,paeth,unser,wronly"
+quiet-level = 3
+
+# to undo: ./test_data/input/LICENSE-3rdparty

run

@@ -700,214 +700,6 @@ download_testdata() {
     fi
 }
 
-launch_notebooks_desc() { echo 'Launch jupyter notebooks
-
-Arguments:
-  -p <port>                - port number
-  -h <host>                - hostname to serve documentation on (default: 0.0.0.0)
-  -g <path to NVMe device> - launch GDS-enabled container
-'
-}
-launch_notebooks() {
-    local OPTIND
-    local port=$(get_unused_ports 1 10000 10030)
-    local host='0.0.0.0'
-    local gds_postfix=''
-    local gds_nvme_path=''
-
-    while getopts 'p:h:g:' option;
-    do
-        case "${option}" in
-            p)
-                port="$OPTARG"
-                ;;
-            h)
-                host="$OPTARG"
-                ;;
-            g)
-                gds_postfix='-gds'
-                [ -z "$OPTARG" ] && c_echo_err R "Please specify NVMe path!" && return 1
-                gds_nvme_path=$(readlink -f "$OPTARG")
-                [ ! -d "$gds_nvme_path" ] && c_echo_err R "Folder $gds_nvme_path doesn't exist!" && return 1
-
-                # Copy cufile SDK from host system to temp/cuda
-                copy_gds_files_
-                ;;
-            *)
-                return 1
-       esac
-    done
-    shift $((OPTIND-1))
-
-    download_testdata
-
-    run_command cp ${TOP}/dist/*.whl ${TOP}/notebooks
-
-    run_command nvidia-docker build -t cucim-jupyter${gds_postfix} -f ${TOP}/docker/Dockerfile-jupyter${gds_postfix}-dev ${TOP}
-
-    [ $? -ne 0 ] && return 1
-
-    c_echo W "Port " G "$port" W " would be used...(" B "http://$(hostname -I | cut -d' ' -f 1):${port}" W ")"
-
-    if [ -z "${gds_postfix}" ]; then
-        run_command nvidia-docker run --gpus all -it --rm \
-            -v ${TOP}/notebooks:/notebooks \
-            -p ${port}:${port} \
-            cucim-jupyter \
-            -c "echo -n 'Enter New Password: '; jupyter lab --ServerApp.password=\"\$(python3 -u -c \"from jupyter_server.auth import passwd;pw=input();print(passwd(pw));\" | egrep 'sha|argon')\" --ServerApp.root_dir=/notebooks --allow-root --port=${port} --ip=${host} --no-browser"
-    else
-        local MNT_PATH=/nvme
-        local GDS_IMAGE=cucim-jupyter${gds_postfix}
-
-        local BUILD_VER=`uname -r`
-        local NV_DRIVER=`nvidia-smi -q -i 0 | sed -n 's/Driver Version.*: *\(.*\) *$/\1/p'`
-        echo "using nvidia driver version $NV_DRIVER on kernel $BUILD_VER"
-
-        local ofed_version=$(ofed_info -s | grep MLNX)
-        if [ $? -eq 0 ]; then
-            local rdma_core=$(dpkg -s libibverbs-dev | grep "Source: rdma-core")
-            if [ $? -eq 0 ]; then
-                local CONFIG_MOFED_VERSION=$(echo $ofed_version | cut -d '-' -f 2)
-                echo "Found MOFED version $CONFIG_MOFED_VERSION"
-            fi
-            local MLNX_SRCS="--volume /usr/src/mlnx-ofed-kernel-${CONFIG_MOFED_VERSION}:/usr/src/mlnx-ofed-kernel-${CONFIG_MOFED_VERSION}:ro"
-            local MOFED_DEVS="--net=host --volume /sys/class/infiniband_verbs:/sys/class/infiniband_verbs/ "
-        fi
-
-        docker run \
-            --ipc host \
-            -it \
-            --rm \
-            --gpus all \
-            -v ${TOP}/notebooks:/notebooks \
-            -p ${port}:${port} \
-            --volume /run/udev:/run/udev:ro \
-            --volume /sys/kernel/config:/sys/kernel/config/ \
-            --volume /usr/src/nvidia-$NV_DRIVER:/usr/src/nvidia-$NV_DRIVER:ro  ${MLNX_SRCS}\
-            --volume /dev:/dev:ro \
-            --privileged \
-            --env NV_DRIVER=${NV_DRIVER} \
-            --volume /lib/modules/$BUILD_VER/:/lib/modules/$BUILD_VER \
-            --volume "${MNT_PATH}:/notebooks/nvme:rw" \
-            ${MOFED_DEVS} \
-            ${GDS_IMAGE} \
-            -c "echo -n 'Enter New Password: '; jupyter lab --ServerApp.password=\"\$(python3 -u -c \"from jupyter_server.auth import passwd;pw=input();print(passwd(pw));\" | egrep 'sha|argon')\" --ServerApp.root_dir=/notebooks --allow-root --port=${port} --ip=${host} --no-browser"
-    fi
-
-}
-
-#==================================================================================
-# Section: Documentation
-#==================================================================================
-
-install_tox_() {
-    if ! command -v tox > /dev/null; then
-        c_echo G "tox" W " doesn't exists. Installing " G "tox" W "..."
-        if [ -n "${CONDA_PREFIX}" ]; then
-            run_command pip3 install tox
-        else
-            if [ -n "${VIRTUAL_ENV}" ]; then
-                run_command pip3 install tox
-            else
-                run_command pip3 install --user tox
-            fi
-        fi
-        hash -r
-    fi
-}
-
-gen_docs_desc() { echo 'Generate document
-
-Generated docs would be available at ${TOP}/dist/docs.
-
-Returns:
-  None
-
-  Exit code:
-    exit code returned from generating document
-'
-}
-gen_docs() {
-    local OUTPUT_FOLDER=${1:-${TOP}/dist/docs}
-    local ret=0
-    pushd ${TOP}/python/cucim > /dev/null
-
-    # Install prerequisites
-    install_tox_
-
-    # Remove existing files in dist/docs
-    run_command rm -rf ${OUTPUT_FOLDER}/*
-
-    # Copy notebook files to python/cucim/dist/docs/notebooks
-    run_command mkdir -p ${TOP}/python/cucim/docs/notebooks
-    run_command rm -rf ${TOP}/python/cucim/docs/notebooks/*
-    run_command mkdir -p ${TOP}/python/cucim/docs/notebooks/static_images
-    run_command cp -r $(git ls-files ${TOP}/notebooks/*.ipynb) ${TOP}/python/cucim/docs/notebooks/
-    run_command cp -r ${TOP}/notebooks/static_images/*.png ${TOP}/python/cucim/docs/notebooks/static_images/
-
-    tox -e docs -- ${OUTPUT_FOLDER}
-    ret=$?
-    # Remove jupyter_execute folder explicitly until the issue is solved
-    #   https://github.com/executablebooks/MyST-NB/issues/129
-    rm -rf $(dirname ${OUTPUT_FOLDER})/jupyter_execute
-
-    popd > /dev/null
-    return $ret
-}
-
-gen_docs_dev_desc() { echo 'Generate document
-
-Launch dev-server for sphinx.
-
-Generated docs would be available at ${TOP}/python/cucim/dist/docs.
-
-Arguments:
-  -p <port> - port number
-  -h <host> - hostname to serve documentation on (default: 0.0.0.0)
-Returns:
-  None
-
-  Exit code:
-    exit code returned from generating document
-'
-}
-gen_docs_dev() {
-    local OPTIND
-    local port=9999
-    local host=0.0.0.0
-
-    while getopts 'p:h:' option;
-    do
-        case "${option}" in
-            p)
-                port="$OPTARG"
-                ;;
-            h)
-                host="$OPTARG"
-                ;;
-            *)
-                echo_err R "Invalid option!"
-                return 1
-        esac
-    done
-
-    pushd ${TOP}/python/cucim > /dev/null
-
-    # Install prerequisites
-    install_tox_
-
-    # Remove existing files in python/cucim/dist/docs
-    run_command rm -rf ${TOP}/python/cucim/dist/docs/*
-    # Copy notebook files to python/cucim/dist/docs/notebooks
-    run_command mkdir -p ${TOP}/python/cucim/docs/notebooks/static_images
-    run_command rm -rf ${TOP}/python/cucim/docs/notebooks/*
-    run_command cp -r $(git ls-files ${TOP}/notebooks/*.ipynb) ${TOP}/python/cucim/docs/notebooks/
-    run_command cp -r ${TOP}/notebooks/static_images/*.png ${TOP}/python/cucim/docs/notebooks/static_images/
-
-    run_command tox -e docs-dev -- --port ${port} --host ${host} docs dist/docs
-    popd > /dev/null
-}
-
 #==================================================================================
 # Section: Release
 #==================================================================================
@@ -934,23 +726,6 @@ update_version() {
     return $ret
 }
 
-import_env_vars_() {
-    if [ ! -e ${TOP}/.env ]; then
-        c_echo_err "File " G "${TOP}/.env " Z "is not found.\n" R \
-        "Please create the file with the following environment variable!"
-        c_echo_err " - " W "GITLAB_ACCESS_TOKEN"
-        c_echo_err " - " W "GITLAB_PUBLISH_SERVER"
-        c_echo_err " - " W "GITLAB_PUBLISH_PROJECT_NAME"
-        c_echo_err " - " W "GITLAB_PUBLISH_PROJECT_ID"
-        c_echo_err " - " W "GITLAB_PUBLISH_PROJECT_URL"
-        c_echo_err " - " W "GITLAB_PUBLISH_GIT_URL"
-        return 1
-    fi
-
-    # Import environment variables
-    . ${TOP}/.env
-}
-
 parse_args() {
     local OPTIND
     while getopts 'yh' option;
@@ -1015,9 +790,6 @@ main() {
             print_cmd_help_messages "${ARGS[@]}"
             exit 0
             ;;
-        notebooks)
-            launch_notebooks "${ARGS[@]}"
-            ;;
         ''|main)
             print_usage
             ;;