diff --git a/examples/UDF-Examples/Spark-cuSpatial/tools/convert_data.sh b/.github/workflows/add-to-project.yml old mode 100755 new mode 100644 similarity index 50% rename from examples/UDF-Examples/Spark-cuSpatial/tools/convert_data.sh rename to .github/workflows/add-to-project.yml index bc189d21f..80f5d8a01 --- a/examples/UDF-Examples/Spark-cuSpatial/tools/convert_data.sh +++ b/.github/workflows/add-to-project.yml @@ -1,5 +1,4 @@ -#!/bin/bash -# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,8 +11,25 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# This script is used to convert a ".cny" points file into parquet. - -python to_parquet.py /data/cuspatial/points.cny /data/cuspatial/points/points.parquet +name: Add new issues and pull requests to project + +on: + issues: + types: + - opened + pull_request_target: + types: + - opened + +jobs: + add-to-project: + if: github.repository == 'NVIDIA/spark-rapids-examples' + name: Add new issues and pull requests to project + runs-on: ubuntu-latest + steps: + - uses: actions/add-to-project@v0.6.1 + with: + project-url: https://github.com/orgs/NVIDIA/projects/4 + github-token: ${{ secrets.PROJECT_TOKEN }} + diff --git a/.github/workflows/auto-merge.yml b/.github/workflows/auto-merge.yml index bab0f3f71..9473dae4e 100644 --- a/.github/workflows/auto-merge.yml +++ b/.github/workflows/auto-merge.yml @@ -18,7 +18,7 @@ name: auto-merge HEAD to BASE on: pull_request_target: branches: - - branch-24.06 + - branch-24.08 types: [closed] jobs: @@ -29,14 +29,14 @@ jobs: steps: - uses: actions/checkout@v4 with: - ref: branch-24.06 # force to fetch from latest upstream instead of PR ref + ref: branch-24.08 # force to fetch from latest upstream instead of PR ref - name: auto-merge job uses: ./.github/workflows/auto-merge env: OWNER: NVIDIA REPO_NAME: spark-rapids-examples - HEAD: branch-24.06 - BASE: branch-24.08 + HEAD: branch-24.08 + BASE: branch-24.10 AUTOMERGE_TOKEN: ${{ secrets.AUTOMERGE_TOKEN }} # use to merge PR diff --git a/README.md b/README.md index a75e6dacc..0a3e9c704 100644 --- a/README.md +++ b/README.md @@ -23,9 +23,7 @@ Here is the list of notebooks in this repo: | 3 | XGBoost | Agaricus (Scala) | Uses XGBoost classifier function to create model that can accurately differentiate between edible and poisonous mushrooms with the [agaricus dataset](https://archive.ics.uci.edu/ml/datasets/mushroom) | 4 | XGBoost | Mortgage (Scala) | End-to-end ETL + XGBoost example to predict mortgage default with [Fannie Mae Single-Family Loan Performance Data](https://capitalmarkets.fanniemae.com/credit-risk-transfer/single-family-credit-risk-transfer/fannie-mae-single-family-loan-performance-data) | 5 | XGBoost | Taxi (Scala) | End-to-end ETL + XGBoost example to predict taxi trip fare amount with [NYC taxi trips data set](https://www1.nyc.gov/site/tlc/about/tlc-trip-record-data.page) -| 6 | ML/DL | Criteo Training | ETL and deep learning training of the Criteo 1TB Click Logs dataset -| 7 | ML/DL | PCA End-to-End | Spark MLlib based PCA example to train and transform with a synthetic dataset -| 8 | UDF | cuSpatial - Point in Polygon | Spark cuSpatial example for Point in Polygon function using NYC Taxi pickup location dataset +| 6 | ML/DL | PCA End-to-End | Spark MLlib based PCA example to train and transform with a synthetic dataset Here is the list of Apache Spark applications (Scala and PySpark) that can be built for running on GPU with RAPIDS Accelerator in this repo: @@ -36,8 +34,7 @@ can be built for running on GPU with RAPIDS Accelerator in this repo: | 2 | XGBoost | Mortgage (Scala) | End-to-end ETL + XGBoost example to predict mortgage default with [Fannie Mae Single-Family Loan Performance Data](https://capitalmarkets.fanniemae.com/credit-risk-transfer/single-family-credit-risk-transfer/fannie-mae-single-family-loan-performance-data) | 3 | XGBoost | Taxi (Scala) | End-to-end ETL + XGBoost example to predict taxi trip fare amount with [NYC taxi trips data set](https://www1.nyc.gov/site/tlc/about/tlc-trip-record-data.page) | 4 | ML/DL | PCA End-to-End | Spark MLlib based PCA example to train and transform with a synthetic dataset -| 5 | UDF | cuSpatial - Point in Polygon | Spark cuSpatial example for Point in Polygon function using NYC Taxi pickup location dataset -| 6 | UDF | URL Decode | Decodes URL-encoded strings using the [Java APIs of RAPIDS cudf](https://docs.rapids.ai/api/cudf-java/legacy/) -| 7 | UDF | URL Encode | URL-encodes strings using the [Java APIs of RAPIDS cudf](https://docs.rapids.ai/api/cudf-java/legacy/) -| 8 | UDF | [CosineSimilarity](./examples/UDF-Examples/RAPIDS-accelerated-UDFs/src/main/java/com/nvidia/spark/rapids/udf/java/CosineSimilarity.java) | Computes the cosine similarity between two float vectors using [native code](./examples/UDF-Examples/RAPIDS-accelerated-UDFs/src/main/cpp/src) -| 9 | UDF | [StringWordCount](./examples/UDF-Examples/RAPIDS-accelerated-UDFs/src/main/java/com/nvidia/spark/rapids/udf/hive/StringWordCount.java) | Implements a Hive simple UDF using [native code](./examples/UDF-Examples/RAPIDS-accelerated-UDFs/src/main/cpp/src) to count words in strings +| 5 | UDF | URL Decode | Decodes URL-encoded strings using the [Java APIs of RAPIDS cudf](https://docs.rapids.ai/api/cudf-java/legacy/) +| 6 | UDF | URL Encode | URL-encodes strings using the [Java APIs of RAPIDS cudf](https://docs.rapids.ai/api/cudf-java/legacy/) +| 7 | UDF | [CosineSimilarity](./examples/UDF-Examples/RAPIDS-accelerated-UDFs/src/main/java/com/nvidia/spark/rapids/udf/java/CosineSimilarity.java) | Computes the cosine similarity between two float vectors using [native code](./examples/UDF-Examples/RAPIDS-accelerated-UDFs/src/main/cpp/src) +| 8 | UDF | [StringWordCount](./examples/UDF-Examples/RAPIDS-accelerated-UDFs/src/main/java/com/nvidia/spark/rapids/udf/hive/StringWordCount.java) | Implements a Hive simple UDF using [native code](./examples/UDF-Examples/RAPIDS-accelerated-UDFs/src/main/cpp/src) to count words in strings diff --git a/datasets/tpcds-small.tar.gz b/datasets/tpcds-small.tar.gz index 065224273..c91120562 100644 Binary files a/datasets/tpcds-small.tar.gz and b/datasets/tpcds-small.tar.gz differ diff --git a/docs/get-started/xgboost-examples/csp/databricks/databricks.md b/docs/get-started/xgboost-examples/csp/databricks/databricks.md index 1377a1532..d56f8ca84 100644 --- a/docs/get-started/xgboost-examples/csp/databricks/databricks.md +++ b/docs/get-started/xgboost-examples/csp/databricks/databricks.md @@ -21,7 +21,7 @@ Navigate to your home directory in the UI and select **Create** > **File** from create an `init.sh` scripts with contents: ```bash #!/bin/bash - sudo wget -O /databricks/jars/rapids-4-spark_2.12-24.06.0.jar https://repo1.maven.org/maven2/com/nvidia/rapids-4-spark_2.12/24.06.0/rapids-4-spark_2.12-24.06.0.jar + sudo wget -O /databricks/jars/rapids-4-spark_2.12-24.08.1.jar https://repo1.maven.org/maven2/com/nvidia/rapids-4-spark_2.12/24.08.1/rapids-4-spark_2.12-24.08.1.jar ``` 1. Select the Databricks Runtime Version from one of the supported runtimes specified in the Prerequisites section. @@ -68,7 +68,7 @@ create an `init.sh` scripts with contents: ```bash spark.rapids.sql.python.gpu.enabled true spark.python.daemon.module rapids.daemon_databricks - spark.executorEnv.PYTHONPATH /databricks/jars/rapids-4-spark_2.12-24.06.0.jar:/databricks/spark/python + spark.executorEnv.PYTHONPATH /databricks/jars/rapids-4-spark_2.12-24.08.1.jar:/databricks/spark/python ``` Note that since python memory pool require installing the cudf library, so you need to install cudf library in each worker nodes `pip install cudf-cu11 --extra-index-url=https://pypi.nvidia.com` or disable python memory pool diff --git a/docs/get-started/xgboost-examples/csp/databricks/init.sh b/docs/get-started/xgboost-examples/csp/databricks/init.sh index 6dbb9052a..750a07335 100644 --- a/docs/get-started/xgboost-examples/csp/databricks/init.sh +++ b/docs/get-started/xgboost-examples/csp/databricks/init.sh @@ -1,7 +1,7 @@ sudo rm -f /databricks/jars/spark--maven-trees--ml--10.x--xgboost-gpu--ml.dmlc--xgboost4j-gpu_2.12--ml.dmlc__xgboost4j-gpu_2.12__1.5.2.jar sudo rm -f /databricks/jars/spark--maven-trees--ml--10.x--xgboost-gpu--ml.dmlc--xgboost4j-spark-gpu_2.12--ml.dmlc__xgboost4j-spark-gpu_2.12__1.5.2.jar -sudo wget -O /databricks/jars/rapids-4-spark_2.12-24.06.0.jar https://repo1.maven.org/maven2/com/nvidia/rapids-4-spark_2.12/24.06.0/rapids-4-spark_2.12-24.06.0.jar +sudo wget -O /databricks/jars/rapids-4-spark_2.12-24.08.1.jar https://repo1.maven.org/maven2/com/nvidia/rapids-4-spark_2.12/24.08.1/rapids-4-spark_2.12-24.08.1.jar sudo wget -O /databricks/jars/xgboost4j-gpu_2.12-1.7.1.jar https://repo1.maven.org/maven2/ml/dmlc/xgboost4j-gpu_2.12/1.7.1/xgboost4j-gpu_2.12-1.7.1.jar sudo wget -O /databricks/jars/xgboost4j-spark-gpu_2.12-1.7.1.jar https://repo1.maven.org/maven2/ml/dmlc/xgboost4j-spark-gpu_2.12/1.7.1/xgboost4j-spark-gpu_2.12-1.7.1.jar ls -ltr diff --git a/docs/get-started/xgboost-examples/on-prem-cluster/kubernetes-scala.md b/docs/get-started/xgboost-examples/on-prem-cluster/kubernetes-scala.md index 6bd7ea155..2d685ef54 100644 --- a/docs/get-started/xgboost-examples/on-prem-cluster/kubernetes-scala.md +++ b/docs/get-started/xgboost-examples/on-prem-cluster/kubernetes-scala.md @@ -40,7 +40,7 @@ export SPARK_DOCKER_IMAGE= export SPARK_DOCKER_TAG= pushd ${SPARK_HOME} -wget https://github.com/NVIDIA/spark-rapids-examples/raw/branch-24.06/dockerfile/Dockerfile +wget https://github.com/NVIDIA/spark-rapids-examples/raw/branch-24.08/dockerfile/Dockerfile # Optionally install additional jars into ${SPARK_HOME}/jars/ diff --git a/docs/get-started/xgboost-examples/prepare-package-data/preparation-python.md b/docs/get-started/xgboost-examples/prepare-package-data/preparation-python.md index e34e0557d..61df8def3 100644 --- a/docs/get-started/xgboost-examples/prepare-package-data/preparation-python.md +++ b/docs/get-started/xgboost-examples/prepare-package-data/preparation-python.md @@ -5,7 +5,7 @@ For simplicity export the location to these jars. All examples assume the packag ### Download the jars Download the RAPIDS Accelerator for Apache Spark plugin jar - * [RAPIDS Spark Package](https://repo1.maven.org/maven2/com/nvidia/rapids-4-spark_2.12/24.06.0/rapids-4-spark_2.12-24.06.0.jar) + * [RAPIDS Spark Package](https://repo1.maven.org/maven2/com/nvidia/rapids-4-spark_2.12/24.08.1/rapids-4-spark_2.12-24.08.1.jar) ### Build XGBoost Python Examples @@ -16,4 +16,4 @@ Following this [guide](/docs/get-started/xgboost-examples/building-sample-apps/p You need to copy the dataset to `/opt/xgboost`. Use the following links to download the data. 1. [Mortgage dataset](/docs/get-started/xgboost-examples/dataset/mortgage.md) 2. [Taxi dataset](https://www1.nyc.gov/site/tlc/about/tlc-trip-record-data.page) -3. [Agaricus dataset](https://gust.dev/r/xgboost-agaricus) +3. [Agaricus dataset](https://github.com/dmlc/xgboost/tree/master/demo/data) diff --git a/docs/get-started/xgboost-examples/prepare-package-data/preparation-scala.md b/docs/get-started/xgboost-examples/prepare-package-data/preparation-scala.md index 5fc42d603..ba4ab38d7 100644 --- a/docs/get-started/xgboost-examples/prepare-package-data/preparation-scala.md +++ b/docs/get-started/xgboost-examples/prepare-package-data/preparation-scala.md @@ -5,7 +5,7 @@ For simplicity export the location to these jars. All examples assume the packag ### Download the jars 1. Download the RAPIDS Accelerator for Apache Spark plugin jar - * [RAPIDS Spark Package](https://repo1.maven.org/maven2/com/nvidia/rapids-4-spark_2.12/24.06.0/rapids-4-spark_2.12-24.06.0.jar) + * [RAPIDS Spark Package](https://repo1.maven.org/maven2/com/nvidia/rapids-4-spark_2.12/24.08.1/rapids-4-spark_2.12-24.08.1.jar) ### Build XGBoost Scala Examples @@ -16,4 +16,4 @@ Following this [guide](/docs/get-started/xgboost-examples/building-sample-apps/s You need to copy the dataset to `/opt/xgboost`. Use the following links to download the data. 1. [Mortgage dataset](/docs/get-started/xgboost-examples/dataset/mortgage.md) 2. [Taxi dataset](https://www1.nyc.gov/site/tlc/about/tlc-trip-record-data.page) -3. [Agaricus dataset](https://gust.dev/r/xgboost-agaricus) +3. [Agaricus dataset](https://github.com/dmlc/xgboost/tree/master/demo/data) diff --git a/docs/img/guides/microbm.png b/docs/img/guides/microbm.png index e04553b37..581c39543 100644 Binary files a/docs/img/guides/microbm.png and b/docs/img/guides/microbm.png differ diff --git a/examples/ML+DL-Examples/Spark-DL/criteo_train/Dockerfile b/examples/ML+DL-Examples/Spark-DL/criteo_train/Dockerfile deleted file mode 100644 index 0e56925b1..000000000 --- a/examples/ML+DL-Examples/Spark-DL/criteo_train/Dockerfile +++ /dev/null @@ -1,229 +0,0 @@ -# -# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -ARG IMAGE=nvcr.io/nvidia/tensorflow:21.03-tf2-py3 -FROM ${IMAGE} -ENV CUDA_SHORT_VERSION=11.2 - -SHELL ["/bin/bash", "-c"] -ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64:/usr/local/lib:/repos/dist/lib - -ENV DEBIAN_FRONTEND=noninteractive - -ARG RELEASE=true -ARG RMM_VER=v21.10.00 -ARG CUDF_VER=v21.10.00 -ARG NVTAB_VER=v0.6.0 -ARG SM="60;61;70;75;80" - -ENV CUDA_HOME=/usr/local/cuda -ENV CUDA_PATH=$CUDA_HOME -ENV CUDA_CUDA_LIBRARY=${CUDA_HOME}/lib64/stubs -ENV PATH=${CUDA_HOME}/lib64/:${PATH}:${CUDA_HOME}/bin - -# Build env variables for rmm -ENV INSTALL_PREFIX=/usr - -RUN apt update -y --fix-missing && \ - apt upgrade -y && \ - apt install -y --no-install-recommends software-properties-common && \ - apt update -y --fix-missing - -RUN apt install -y --no-install-recommends \ - git \ - libboost-all-dev \ - python3.8-dev \ - build-essential \ - autoconf \ - bison \ - flex \ - libboost-filesystem-dev \ - libboost-system-dev \ - libboost-regex-dev \ - libjemalloc-dev \ - wget \ - libssl-dev \ - protobuf-compiler \ - clang-format \ - aptitude \ - numactl \ - libnuma-dev \ - libaio-dev \ - libibverbs-dev \ - libtool && \ - apt-get autoremove -y && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - #update-alternatives --install /usr/bin/python python /usr/bin/python3.8 1 && \ - #wget https://bootstrap.pypa.io/get-pip.py && \ - #python get-pip.py - -# Install cmake -RUN wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null && \ - apt-add-repository 'deb https://apt.kitware.com/ubuntu/ focal main' && \ - apt-get update && \ - apt-get install -y cmake - -# Install arrow from source -ENV ARROW_HOME=/usr/local -RUN git clone --branch apache-arrow-4.0.1 --recurse-submodules https://github.com/apache/arrow.git build-env && \ - pushd build-env && \ - export PARQUET_TEST_DATA="${PWD}/cpp/submodules/parquet-testing/data" && \ - export ARROW_TEST_DATA="${PWD}/testing/data" && \ - pip install -r python/requirements-build.txt && \ - mkdir cpp/release && \ - pushd cpp/release && \ - cmake -DCMAKE_INSTALL_PREFIX=${ARROW_HOME} \ - -DCMAKE_INSTALL_LIBDIR=lib \ - -DCMAKE_LIBRARY_PATH=${CUDA_CUDA_LIBRARY} \ - -DARROW_FLIGHT=ON \ - -DARROW_GANDIVA=OFF \ - -DARROW_ORC=ON \ - -DARROW_WITH_BZ2=ON \ - -DARROW_WITH_ZLIB=ON \ - -DARROW_WITH_ZSTD=ON \ - -DARROW_WITH_LZ4=ON \ - -DARROW_WITH_SNAPPY=ON \ - -DARROW_WITH_BROTLI=ON \ - -DARROW_PARQUET=ON \ - -DARROW_PYTHON=ON \ - -DARROW_PLASMA=ON \ - -DARROW_BUILD_TESTS=ON \ - -DARROW_CUDA=ON \ - -DARROW_DATASET=ON \ - .. && \ - make -j$(nproc) && \ - make install && \ - popd && \ - pushd python && \ - export PYARROW_WITH_PARQUET=ON && \ - export PYARROW_WITH_CUDA=ON && \ - export PYARROW_WITH_ORC=ON && \ - export PYARROW_WITH_DATASET=ON && \ - python setup.py build_ext --build-type=release bdist_wheel && \ - pip install dist/*.whl && \ - popd && \ - popd && \ - rm -rf build-env - - -# Install rmm from source -RUN git clone https://github.com/rapidsai/rmm.git build-env && cd build-env/ && \ - if [ "$RELEASE" == "true" ] && [ ${RMM_VER} != "vnightly" ] ; then git fetch --all --tags && git checkout tags/${RMM_VER}; else git checkout main; fi; \ - sed -i '/11.2/ a "11.4": "11.x",' python/setup.py && \ - cd ..; \ - pushd build-env && \ - ./build.sh librmm && \ - pip install python/. && \ - popd && \ - rm -rf build-env - - - -# Build env for CUDF build -RUN git clone https://github.com/rapidsai/cudf.git build-env && cd build-env/ && \ - if [ "$RELEASE" == "true" ] && [ ${CUDF_VER} != "vnightly" ] ; then git fetch --all --tags && git checkout tags/${CUDF_VER}; else git checkout main; fi; \ - git submodule update --init --recursive && \ - cd .. && \ - pushd build-env && \ - export CUDF_HOME=${PWD} && \ - export CUDF_ROOT=${PWD}/cpp/build/ && \ - export CMAKE_LIBRARY_PATH=${CUDA_CUDA_LIBRARY} && \ - ./build.sh libcudf cudf dask_cudf && \ - protoc -I=python/cudf/cudf/utils/metadata --python_out=/usr/local/lib/python3.8/dist-packages/cudf/utils/metadata python/cudf/cudf/utils/metadata/orc_column_statistics.proto && \ - popd && \ - rm -rf build-env - -RUN apt-get update -y && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - vim gdb git wget unzip tar python3.8-dev \ - zlib1g-dev lsb-release clang-format libboost-all-dev \ - openssl curl zip\ - slapd && \ - rm -rf /var/lib/apt/lists/* - -ENV PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION='python' - - -RUN pip install pandas sklearn ortools nvtx-plugins pydot && \ - pip cache purge - -# tf-nightly for performance test -# more details: https://github.com/tensorflow/tensorflow/issues/44194 -RUN pip uninstall tensorflow -y; pip install tf-nightly==2.7.0.dev20210722 -RUN pip uninstall keras-nightly -y; pip install keras-nightly==2.7.0.dev2021072200 - - -RUN mkdir -p /usr/local/nvidia/lib64 && \ - ln -s /usr/local/cuda/lib64/libcusolver.so /usr/local/nvidia/lib64/libcusolver.so.10 - -RUN pip install pybind11 -SHELL ["/bin/bash", "-c"] - -# prepare nccl -RUN apt remove -y libnccl2 libnccl-dev -RUN wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-ubuntu2004.pin \ - && mv cuda-ubuntu2004.pin /etc/apt/preferences.d/cuda-repository-pin-600 \ - && apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/7fa2af80.pub \ - && add-apt-repository "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/ /" \ - && apt-get update \ - && apt install libnccl2=2.8.4-1+cuda11.2 libnccl-dev=2.8.4-1+cuda11.2 - -# install Horovod -RUN pip uninstall horovod -y -RUN HOROVOD_WITH_MPI=1 HOROVOD_WITH_TENSORFLOW=1 HOROVOD_GPU_OPERATIONS=NCCL \ - pip install horovod[spark] --no-cache-dir - -# Install NVTabular -RUN git clone https://github.com/NVIDIA/NVTabular.git /nvtabular/ && \ - cd /nvtabular/; if [ "$RELEASE" == "true" ] && [ ${NVTAB_VER} != "vnightly" ] ; then git fetch --all --tags && git checkout tags/${NVTAB_VER}; else git checkout main; fi; \ - python setup.py develop --user; - - -RUN pip install pynvml pytest graphviz sklearn scipy matplotlib -RUN pip install nvidia-pyindex; pip install tritonclient[all] grpcio-channelz -RUN pip install nvtx mpi4py==3.0.3 cupy-cuda112 cachetools typing_extensions fastavro - -RUN apt-get update; apt-get install -y graphviz - -RUN pip uninstall numpy -y; pip install numpy -RUN pip install dask==2021.04.1 distributed==2021.04.1 dask-cuda -RUN pip install dask[dataframe]==2021.04.1 -RUN pip uninstall pandas -y; pip install pandas==1.1.5 -RUN echo $(du -h --max-depth=1 /) - - -# install spark-3.1.2-bin-hadoop3.2 -RUN wget \ - https://mirror-hk.koddos.net/apache/spark/spark-3.1.2/spark-3.1.2-bin-hadoop3.2.tgz \ - && tar -xzf spark-3.1.2-bin-hadoop3.2.tgz -C /opt/ \ - && rm spark-3.1.2-bin-hadoop3.2.tgz - -ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get install openjdk-8-jdk openjdk-8-jre lsb-release -y --allow-downgrades --allow-change-held-packages --no-install-recommends -ENV JAVA_HOME /usr/lib/jvm/java-1.8.0-openjdk-amd64 -ENV PATH $PATH:/usr/lib/jvm/java-1.8.0-openjdk-amd64/jre/bin:/usr/lib/jvm/java-1.8.0-openjdk-amd64/bin - -# add spark env to conf -ADD spark-env.sh /opt/spark-3.1.2-bin-hadoop3.2/conf/ - -ADD start-spark.sh /workspace/ -ADD submit.sh /workspace/ -ADD criteo_keras.py /workspace/ - -HEALTHCHECK NONE -ENTRYPOINT [] -CMD ["/bin/bash"] diff --git a/examples/ML+DL-Examples/Spark-DL/criteo_train/Dockerfile.conda_db b/examples/ML+DL-Examples/Spark-DL/criteo_train/Dockerfile.conda_db deleted file mode 100644 index 475b99149..000000000 --- a/examples/ML+DL-Examples/Spark-DL/criteo_train/Dockerfile.conda_db +++ /dev/null @@ -1,78 +0,0 @@ -# -# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -FROM nvidia/cuda:11.4.3-cudnn8-devel-ubuntu20.04 - -ENV DEBIAN_FRONTEND=noninteractive -# Disable NVIDIA repos to prevent accidental upgrades. -RUN cd /etc/apt/sources.list.d && \ - mv cuda.list cuda.list.disabled - -# See https://github.com/databricks/containers/blob/master/ubuntu/minimal/Dockerfile -RUN apt-get update && \ - apt-get install --yes --no-install-recommends \ - openjdk-8-jdk \ - openjdk-8-jre \ - lsb-release \ - iproute2 \ - bash \ - sudo \ - coreutils \ - procps \ - wget && \ - /var/lib/dpkg/info/ca-certificates-java.postinst configure && \ - rm -rf /var/lib/apt/lists/* - - -ENV PATH /databricks/conda/bin:$PATH - -RUN wget -q https://repo.continuum.io/miniconda/Miniconda3-py38_4.9.2-Linux-x86_64.sh -O miniconda.sh && \ - bash miniconda.sh -b -p /databricks/conda && \ - rm miniconda.sh && \ - # Source conda.sh for all login and interactive shells. - ln -s /databricks/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \ - echo ". /etc/profile.d/conda.sh" >> ~/.bashrc && \ - # Set always_yes for non-interactive shells. - conda config --system --set always_yes True && \ - conda clean --all - -# install openjdk8, cmake, openmpi openmpi-mpicc -RUN conda install cmake openmpi openmpi-mpicc -y -ENV JAVA_HOME /usr/lib/jvm/java-1.8.0-openjdk-amd64 -ENV PATH $PATH:/usr/lib/jvm/java-1.8.0-openjdk-amd64/jre/bin:/usr/lib/jvm/java-1.8.0-openjdk-amd64/bin - -RUN conda install -y -c nvidia -c rapidsai -c numba -c conda-forge nvtabular=1.2.2 python=3.8 cudatoolkit=11.4 scikit-learn - -RUN pip uninstall tensorflow -y; pip install tensorflow-gpu==2.8 -RUN pip install torch==1.11.0+cu115 torchvision==0.12.0+cu115 torchaudio===0.11.0+cu115 -f https://download.pytorch.org/whl/cu115/torch_stable.html -RUN rm -rf /databricks/conda/include/google -RUN HOROVOD_WITH_MPI=1 HOROVOD_GPU_OPERATIONS=NCCL HOROVOD_WITH_TENSORFLOW=1 HOROVOD_WITH_PYTORCH=1 \ - pip install horovod[spark] --no-cache-dir -RUN pip install pynvml jupyter matplotlib - - -RUN apt-get update && apt-get install wget openssh-client openssh-server \ - -y --allow-downgrades --allow-change-held-packages --no-install-recommends -RUN useradd --create-home --shell /bin/bash --groups sudo ubuntu - -ENV PYSPARK_PYTHON=/databricks/conda/bin/python -ENV USER root -ENV DEFAULT_DATABRICKS_ROOT_CONDA_ENV=base -ENV DATABRICKS_ROOT_CONDA_ENV=base -# disable gds due to errors -ENV LIBCUDF_CUFILE_POLICY=OFF -# required by DB -RUN pip install virtualenv -RUN pip install adlfs diff --git a/examples/ML+DL-Examples/Spark-DL/criteo_train/README.md b/examples/ML+DL-Examples/Spark-DL/criteo_train/README.md deleted file mode 100644 index 083894055..000000000 --- a/examples/ML+DL-Examples/Spark-DL/criteo_train/README.md +++ /dev/null @@ -1,101 +0,0 @@ -# Criteo - -ETL and deep learning training of the Criteo 1TB Click Logs dataset. Users can prepare their dataset accordingly. - -_Please note: The following demo is dedicated for DGX-2 machine(with V100 GPUs)._ We optimized the whole workflow on DGX-2 and it's not guaranteed that it can run successfully on other type of machines. - -## Dataset - -The dataset used here is from Criteo clicklog dataset. -It's preprocessed by [DLRM](https://github.com/NVIDIA/DeepLearningExamples/tree/master/TensorFlow2/Recommendation/DLRM_and_DCNv2/preproc) -ETL job on Spark. We also provide a small size sample data in sample_data folder. -All 40 columns(1 label + 39 features) are already numeric. - -In the following parts, we assume that the data are mounted as docker volumn at `/data/parquet` - -## Run Criteo example benchmark using Dockerfile - -1. Build the docker image -``` -nvidia-docker build -f Dockerfile -t nvspark/tf-hvd-train:0.1 . -``` - -2. Enter into it (also mount necessary dataset volume and devices) -``` - nvidia-docker run \ - --network host \ - --device /dev/infiniband \ - --privileged \ - -v /raid/spark-team/criteo/parquet:/data/parquet \ - -it nvspark/tf-hvd-train:0.1 bash -``` - -3. when you are inside the container -``` -cd /workspace -# start standalone Spark -./start-spark.sh - -# start training -./submit.sh -``` - -## Notebook demo - -We also provide a Notebook demo for quick test, user can set it up by the following command: - -```bash -SPARK_HOME= $PATH_TO_SPARK_HOME -SPARK_URL=spark://$SPARK_MASTER_IP:7077 -export PYSPARK_DRIVER_PYTHON=jupyter -export PYSPARK_DRIVER_PYTHON_OPTS='notebook' - -$SPARK_HOME/bin/pyspark --master $SPARK_URL --deploy-mode client \ ---driver-memory 20G \ ---executor-memory 50G \ ---executor-cores 6 \ ---conf spark.cores.max=96 \ ---conf spark.task.cpus=6 \ ---conf spark.locality.wait=0 \ ---conf spark.yarn.maxAppAttempts=1 \ ---conf spark.sql.shuffle.partitions=4 \ ---conf spark.sql.files.maxPartitionBytes=1024m \ ---conf spark.sql.warehouse.dir=$OUT \ ---conf spark.task.resource.gpu.amount=0.08 \ ---conf spark.executor.resource.gpu.amount=1 \ ---conf spark.executor.resource.gpu.discoveryScript=./getGpusResources.sh \ ---files $SPARK_HOME/examples/src/main/scripts/getGpusResources.sh - -``` - -## Note: - -If you want to try in a node with only 1 GPU, please modify the GPU number per worker in `$SPARK_HOME/conf/spark-env.sh` before you launch spark workers becasue the docker image is targeted for DGX-2 with 16 GPUs - -## file decription: - -`Dockerfile`: consistent environment, main components are build from source directly. But this file take a while to build an image. - -`spark-env.sh`: Spark config changes, we set 16 GPU for a work for DGX-2 box. It is in SPARK_HOME/conf/ - -`start-spark.sh`: launch Spark in standalone mode. In a DGX-2 box, it will launch Spark Master and a Spark Worker which contains 16 GPUs - -`submit.sh`: commands used to submit the job - -`criteo_keras.py`: Python script to train the Criteo model. Please run `python criteo_keras.py --help` to see parameter details - -## workspace folder in docker container: - -`/workspace/` - -## Run in Databricks Runtime -Some extra packages are required to run the example, we provide a Dockerfile [Dockerfile.conda_db](Dockerfile.conda_db) -to use [Customize containers with Databricks Container Services](https://docs.databricks.com/clusters/custom-containers.html) -in Databricks cloud environment. - -To use it: -1. build the docker image locally -2. push the image to a DB supported Docker hub. -3. set the image url in DB cluster setup page. - -![microbenchmark-speedup](/docs/img/guides/criteo-perf.png) \ No newline at end of file diff --git a/examples/ML+DL-Examples/Spark-DL/criteo_train/create-conda-env.sh b/examples/ML+DL-Examples/Spark-DL/criteo_train/create-conda-env.sh deleted file mode 100755 index f3ba315f3..000000000 --- a/examples/ML+DL-Examples/Spark-DL/criteo_train/create-conda-env.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash --login - -export HOROVOD_CUDA_HOME=/usr/local/cuda -export HOROVOD_NCCL_LINK=SHARED -export HOROVOD_GPU_OPERATIONS=NCCL - -# Create the conda environment: -conda env create --file environment.yml --force - -# Activate the conda environment: -eval "$(conda shell.bash hook)" -conda activate spark_dl - -# Install tensorflow: -pip install tensorflow-gpu - -# Install pytorch: -pip install torch==1.10.2+cu113 torchvision==0.11.3+cu113 torchaudio==0.10.2+cu113\ - -f https://download.pytorch.org/whl/cu113/torch_stable.html - -# Install horovod: -pip install horovod[tensorflow,keras,pytorch,spark] - -# Check the build: -horovodrun --check-build - -echo -echo "Run 'conda activate spark_dl' to activate the environment" -echo diff --git a/examples/ML+DL-Examples/Spark-DL/criteo_train/criteo_keras.py b/examples/ML+DL-Examples/Spark-DL/criteo_train/criteo_keras.py deleted file mode 100644 index bb653f9e6..000000000 --- a/examples/ML+DL-Examples/Spark-DL/criteo_train/criteo_keras.py +++ /dev/null @@ -1,309 +0,0 @@ -# -# Copyright (c) 2021-2021, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import argparse -import math -import pprint -import sys - -# This needs to happen first to avoid pyarrow serialization errors. -from pyspark.sql import SparkSession - -# Make sure pyarrow is referenced before anything else to avoid segfault due to conflict -# with TensorFlow libraries. Use `pa` package reference to ensure it's loaded before -# functions like `deserialize_model` which are implemented at the top level. -# See https://jira.apache.org/jira/browse/ARROW-3346 -import pyarrow as pa - -import horovod -import horovod.tensorflow.keras as hvd -import tensorflow as tf -from horovod.spark.common.backend import SparkBackend -from tensorflow.keras.layers import BatchNormalization, Input, Embedding, Concatenate, Dense, Flatten -from tensorflow.keras.layers.experimental.preprocessing import CategoryEncoding - -PETASTORM_DATALOADER = 'petastorm' -NVTABULAR_DATALOADER = 'nvtabular' - -CONTINUOUS_COLUMNS = [f'i{i}' for i in range(13)] -CATEGORICAL_COLUMNS = [f'c{c}' for c in range(26)] -ALL_COLUMNS = CONTINUOUS_COLUMNS + CATEGORICAL_COLUMNS -LABEL_COLUMNS = ['clicked'] - - -def get_category_dimensions(spark, data_dir): - df = spark.read.csv(f'{data_dir}/dimensions/*.csv', header=True).toPandas() - dimensions = df.to_dict('records')[0] - pprint.pprint(dimensions) - return dimensions - - -def build_model(dimensions, args): - inputs = { - **{i: Input(shape=(1,), name=i, dtype=tf.float32) for i in CONTINUOUS_COLUMNS}, - **{c: Input(shape=(1,), name=c, dtype=tf.int32) for c in CATEGORICAL_COLUMNS} - } - - one_hots = [] - embeddings = [] - for c in CATEGORICAL_COLUMNS: - dimension = int(dimensions[c]) + 1 - if dimension <= 128: - one_hots.append(CategoryEncoding(num_tokens=dimension, name=f'one_hot_{c}')(inputs[c])) - else: - embedding_size = int(math.floor(0.6 * dimension ** 0.25)) - embeddings.append(Embedding(input_dim=dimension, - output_dim=embedding_size, - input_length=1, - name=f'embedding_{c}')(inputs[c])) - - x = Concatenate(name='embeddings_concat')(embeddings) - x = Flatten(name='embeddings_flatten')(x) - x = Concatenate(name='inputs_concat')([x] + one_hots + [inputs[i] for i in CONTINUOUS_COLUMNS]) - x = BatchNormalization()(x) - x = Dense(1024, activation='relu')(x) - x = BatchNormalization()(x) - x = Dense(1024, activation='relu')(x) - x = BatchNormalization()(x) - x = Dense(1024, activation='relu')(x) - x = BatchNormalization()(x) - x = Dense(512, activation='relu')(x) - output = Dense(1, activation='sigmoid', name='output')(x) - model = tf.keras.Model(inputs=[inputs[c] for c in ALL_COLUMNS], outputs=output) - if hvd.rank() == 0: - model.summary() - - opt = tf.keras.optimizers.Adam(learning_rate=args.learning_rate) - opt = hvd.DistributedOptimizer(opt) - model.compile(optimizer=opt, loss='binary_crossentropy', metrics=[tf.keras.metrics.AUC()]) - - return model - - -def train_fn(dimensions, train_rows, val_rows, args): - # Make sure pyarrow is referenced before anything else to avoid segfault due to conflict - # with TensorFlow libraries. Use `pa` package reference to ensure it's loaded before - # functions like `deserialize_model` which are implemented at the top level. - # See https://jira.apache.org/jira/browse/ARROW-3346 - pa - - import atexit - import horovod.tensorflow.keras as hvd - from horovod.spark.task import get_available_devices - import os - import tempfile - import tensorflow as tf - import tensorflow.keras.backend as K - import shutil - - gpus = get_available_devices() - if gpus: - os.environ['CUDA_VISIBLE_DEVICES'] = gpus[0] - if args.dataloader == NVTABULAR_DATALOADER: - os.environ['TF_MEMORY_ALLOCATION'] = '0.85' - from nvtabular.loader.tensorflow import KerasSequenceLoader - - # Horovod: initialize Horovod inside the trainer. - hvd.init() - - # Horovod: restore from checkpoint, use hvd.load_model under the hood. - model = build_model(dimensions, args) - - # Horovod: adjust learning rate based on number of processes. - scaled_lr = K.get_value(model.optimizer.lr) * hvd.size() - K.set_value(model.optimizer.lr, scaled_lr) - - # Horovod: print summary logs on the first worker. - verbose = 1 if hvd.rank() == 0 else 0 - - callbacks = [ - # Horovod: broadcast initial variable states from rank 0 to all other processes. - # This is necessary to ensure consistent initialization of all workers when - # training is started with random weights or restored from a checkpoint. - hvd.callbacks.BroadcastGlobalVariablesCallback(root_rank=0), - - # Horovod: average metrics among workers at the end of every epoch. - # - # Note: This callback must be in the list before the ReduceLROnPlateau, - # TensorBoard, or other metrics-based callbacks. - hvd.callbacks.MetricAverageCallback(), - - # Horovod: using `lr = 1.0 * hvd.size()` from the very beginning leads to worse final - # accuracy. Scale the learning rate `lr = 1.0` ---> `lr = 1.0 * hvd.size()` during - # the first five epochs. See https://arxiv.org/abs/1706.02677 for details. - hvd.callbacks.LearningRateWarmupCallback(initial_lr=scaled_lr, warmup_epochs=5, verbose=verbose), - - # Reduce LR if the metric is not improved for 10 epochs, and stop training - # if it has not improved for 20 epochs. - tf.keras.callbacks.ReduceLROnPlateau(monitor='val_auc', patience=10, verbose=verbose), - tf.keras.callbacks.EarlyStopping(monitor='val_auc', mode='min', patience=20, verbose=verbose), - tf.keras.callbacks.TerminateOnNaN(), - - # Log Tensorboard events. - tf.keras.callbacks.TensorBoard(log_dir=args.logs_dir, write_steps_per_second=True, update_freq=10) - ] - - # Horovod: save checkpoints only on the first worker to prevent other workers from corrupting them. - if hvd.rank() == 0: - ckpt_dir = tempfile.mkdtemp() - ckpt_file = os.path.join(ckpt_dir, 'checkpoint.h5') - atexit.register(lambda: shutil.rmtree(ckpt_dir)) - callbacks.append(tf.keras.callbacks.ModelCheckpoint( - ckpt_file, monitor='val_auc', mode='min', save_best_only=True)) - - if args.dataloader == PETASTORM_DATALOADER: - from petastorm import make_batch_reader - from petastorm.tf_utils import make_petastorm_dataset - - # Make Petastorm readers. - with make_batch_reader(f'{args.data_dir}/train', - num_epochs=None, - cur_shard=hvd.rank(), - shard_count=hvd.size(), - hdfs_driver='libhdfs') as train_reader: - with make_batch_reader(f'{args.data_dir}/val', - num_epochs=None, - cur_shard=hvd.rank(), - shard_count=hvd.size(), - hdfs_driver='libhdfs') as val_reader: - # Convert readers to tf.data.Dataset. - train_ds = make_petastorm_dataset(train_reader) \ - .unbatch() \ - .shuffle(10 * args.batch_size) \ - .batch(args.batch_size) \ - .map(lambda x: (tuple(getattr(x, c) for c in ALL_COLUMNS), x.clicked)) - - val_ds = make_petastorm_dataset(val_reader) \ - .unbatch() \ - .batch(args.batch_size) \ - .map(lambda x: (tuple(getattr(x, c) for c in ALL_COLUMNS), x.clicked)) - - history = model.fit(train_ds, - validation_data=val_ds, - steps_per_epoch=int(train_rows / args.batch_size / hvd.size()), - validation_steps=int(val_rows / args.batch_size / hvd.size()), - callbacks=callbacks, - verbose=verbose, - epochs=args.epochs) - - else: - import cupy - - def seed_fn(): - """ - Generate consistent dataloader shuffle seeds across workers - Reseeds each worker's dataloader each epoch to get fresh a shuffle - that's consistent across workers. - """ - min_int, max_int = tf.int32.limits - max_rand = max_int // hvd.size() - # Generate a seed fragment on each worker - seed_fragment = cupy.random.randint(0, max_rand).get() - # Aggregate seed fragments from all Horovod workers - seed_tensor = tf.constant(seed_fragment) - reduced_seed = hvd.allreduce(seed_tensor, name="shuffle_seed", op=hvd.Sum) - return reduced_seed % max_rand - - train_ds = KerasSequenceLoader( - f'{args.data_dir}/train', - batch_size=args.batch_size, - label_names=LABEL_COLUMNS, - cat_names=CATEGORICAL_COLUMNS, - cont_names=CONTINUOUS_COLUMNS, - engine="parquet", - shuffle=True, - buffer_size=0.06, # how many batches to load at once - parts_per_chunk=1, - global_size=hvd.size(), - global_rank=hvd.rank(), - seed_fn=seed_fn) - - val_ds = KerasSequenceLoader( - f'{args.data_dir}/val', - batch_size=args.batch_size, - label_names=LABEL_COLUMNS, - cat_names=CATEGORICAL_COLUMNS, - cont_names=CONTINUOUS_COLUMNS, - engine="parquet", - shuffle=False, - buffer_size=0.06, # how many batches to load at once - parts_per_chunk=1, - global_size=hvd.size(), - global_rank=hvd.rank()) - - history = model.fit(train_ds, - validation_data=val_ds, - steps_per_epoch=int(train_rows / args.batch_size / hvd.size()), - validation_steps=int(val_rows / args.batch_size / hvd.size()), - callbacks=callbacks, - verbose=verbose, - epochs=args.epochs) - - if hvd.rank() == 0: - return history.history - - -def train(dimensions, train_rows, val_rows, args): - # Horovod: run training. - history = horovod.spark.run(train_fn, - args=(dimensions, train_rows, val_rows, args), - num_proc=args.num_proc, - extra_mpi_args='-mca btl_tcp_if_include enp134s0f0 -x NCCL_IB_GID_INDEX=3', - stdout=sys.stdout, - stderr=sys.stderr, - verbose=2, - nics={}, - prefix_output_with_timestamp=True)[0] - - best_val_loss = min(history['val_loss']) - print('Best Loss: %f' % best_val_loss) - - -def main(): - parser = argparse.ArgumentParser(description='Criteo Spark Keras Training Example', - formatter_class=argparse.ArgumentDefaultsHelpFormatter) - parser.add_argument('--data-dir', default='file:///opt/data/criteo/parquet', - help='location of the transformed Criteo dataset in Parquet format') - parser.add_argument('--logs-dir', default='/opt/experiments/criteo', help='location of TensorFlow logs') - parser.add_argument('--dataloader', default=PETASTORM_DATALOADER, - choices=[PETASTORM_DATALOADER, NVTABULAR_DATALOADER], - help='dataloader to use') - parser.add_argument('--num-proc', type=int, default=1, help='number of worker processes for training') - parser.add_argument('--learning-rate', type=float, default=0.0001, help='initial learning rate') - parser.add_argument('--batch-size', type=int, default=64 * 1024, help='batch size') - parser.add_argument('--epochs', type=int, default=3, help='number of epochs to train') - parser.add_argument('--local-checkpoint-file', default='checkpoint', help='model checkpoint') - args = parser.parse_args() - - spark = SparkSession.builder.appName('Criteo Keras Training').getOrCreate() - - dimensions = get_category_dimensions(spark, args.data_dir) - - train_df = spark.read.parquet(f'{args.data_dir}/train') - val_df = spark.read.parquet(f'{args.data_dir}/val') - test_df = spark.read.parquet(f'{args.data_dir}/test') - train_rows, val_rows, test_rows = train_df.count(), val_df.count(), test_df.count() - print('Training: %d' % train_rows) - print('Validation: %d' % val_rows) - print('Test: %d' % test_rows) - - train(dimensions, train_rows, val_rows, args) - - spark.stop() - - -if __name__ == '__main__': - main() diff --git a/examples/ML+DL-Examples/Spark-DL/criteo_train/environment.yml b/examples/ML+DL-Examples/Spark-DL/criteo_train/environment.yml deleted file mode 100644 index 9bb5e3a52..000000000 --- a/examples/ML+DL-Examples/Spark-DL/criteo_train/environment.yml +++ /dev/null @@ -1,30 +0,0 @@ -name: spark_dl -channels: - - rapidsai - - nvidia - - pytorch - - conda-forge -dependencies: - - nvtabular - - python>=3.7 - - cudatoolkit=11.2 - - cudf>=21.10.* - - dask-cuda>=21.10.* - - dask-cudf>=21.10.* - - rmm>=21.10.* - - dask==2021.9.1 - - distributed>=2021.9.1 - - fsspec>=0.6.0 - - nvtx>=0.2.1 - - numba>=0.53.0 - - dlpack - - scikit-learn - - asvdb - - pynvml - - cmake - - cudnn - - cxx-compiler - - nccl - - nvcc_linux-64 - - openmpi - - pip diff --git a/examples/ML+DL-Examples/Spark-DL/criteo_train/notebooks/Criteo-Training.ipynb b/examples/ML+DL-Examples/Spark-DL/criteo_train/notebooks/Criteo-Training.ipynb deleted file mode 100644 index d760106b4..000000000 --- a/examples/ML+DL-Examples/Spark-DL/criteo_train/notebooks/Criteo-Training.ipynb +++ /dev/null @@ -1,12675 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "dd4592b6", - "metadata": {}, - "source": [ - "# Introduction\n", - "\n", - "This notebook contains the same content as \"criteo_keras.py\" but in a notebook(interactive) form.\n", - "\n", - "The dataset used here is from Criteo clicklog dataset. It's preprocessed by DLRM(https://github.com/NVIDIA/DeepLearningExamples/tree/master/TensorFlow2/Recommendation/DLRM_and_DCNv2/preproc) ETL job on Spark.\n", - "\n", - "We provide a small size sample data in `sample_data` folder.\n", - "\n", - "The data schema after the DLRM ETL: also 40 columns -- 1 label column, 39 numerical feature columns. (Categorical columns are processed)\n", - "\n", - "User can either take the same routine to get the data or use your own preprocessed data and change the DL model accordingly.\n", - "\n", - "`Please note: The following demo is dedicated for DGX-2 machine(with V100 GPUs).` We optimized the whole workflow on DGX-2 and it's not guaranteed that it can run successfully on other type of machines." - ] - }, - { - "cell_type": "markdown", - "id": "e8bdbe10", - "metadata": {}, - "source": [ - "### import necessary libraries" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "cd1fb946", - "metadata": {}, - "outputs": [], - "source": [ - "import argparse\n", - "import math\n", - "import pprint\n", - "import sys\n", - "# This needs to happen first to avoid pyarrow serialization errors.\n", - "from pyspark.sql import SparkSession\n", - "\n", - "# Make sure pyarrow is referenced before anything else to avoid segfault due to conflict\n", - "# with TensorFlow libraries. Use `pa` package reference to ensure it's loaded before\n", - "# functions like `deserialize_model` which are implemented at the top level.\n", - "# See https://jira.apache.org/jira/browse/ARROW-3346\n", - "import pyarrow as pa\n", - "\n", - "import horovod\n", - "import horovod.tensorflow.keras as hvd\n", - "import tensorflow as tf\n", - "from horovod.spark.common.backend import SparkBackend\n", - "from tensorflow.keras.layers import BatchNormalization, Input, Embedding, Concatenate, Dense, Flatten\n", - "from tensorflow.keras.layers.experimental.preprocessing import CategoryEncoding" - ] - }, - { - "cell_type": "markdown", - "id": "ffda2b11", - "metadata": {}, - "source": [ - "### set some macros\n", - "\n", - "xxx_DATALOADER is the switch to control which dataloader we will use,\n", - "xxx_COLUMNS are only used in NVTabular dataloader." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "d690dd8b", - "metadata": {}, - "outputs": [], - "source": [ - "PETASTORM_DATALOADER = 'petastorm'\n", - "NVTABULAR_DATALOADER = 'nvtabular'\n", - "\n", - "CONTINUOUS_COLUMNS = [f'i{i}' for i in range(13)]\n", - "CATEGORICAL_COLUMNS = [f'c{c}' for c in range(26)]\n", - "ALL_COLUMNS = CONTINUOUS_COLUMNS + CATEGORICAL_COLUMNS\n", - "LABEL_COLUMNS = ['clicked']" - ] - }, - { - "cell_type": "markdown", - "id": "cdd51a2a", - "metadata": {}, - "source": [ - "The \"dimensions\" contains the count of distinct value for those categorical column 'after' DLRM ETL.\n", - "\n", - "This will be used to build embedding layers for our model." - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "92e6d359", - "metadata": {}, - "outputs": [], - "source": [ - "def get_category_dimensions(spark, data_dir):\n", - " df = spark.read.csv(f'{data_dir}/dimensions/*.csv', header=True).toPandas()\n", - " dimensions = df.to_dict('records')[0]\n", - " pprint.pprint(dimensions)\n", - " return dimensions" - ] - }, - { - "cell_type": "markdown", - "id": "66e38f1a", - "metadata": {}, - "source": [ - "### Build the model\n", - "\n", - "The model is composed by mainly 2 parts: embedding layers and fully-connected layers." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "0c29876c", - "metadata": {}, - "outputs": [], - "source": [ - "def build_model(dimensions, args):\n", - " \n", - " inputs = {\n", - " **{i: Input(shape=(1,), name=i, dtype=tf.float32) for i in CONTINUOUS_COLUMNS},\n", - " **{c: Input(shape=(1,), name=c, dtype=tf.int32) for c in CATEGORICAL_COLUMNS}\n", - " }\n", - "\n", - " one_hots = []\n", - " embeddings = []\n", - " for c in CATEGORICAL_COLUMNS:\n", - " dimension = int(dimensions[c]) + 1\n", - " # dimension <= 128, smaller size for demo\n", - " if dimension <= 8:\n", - " one_hots.append(CategoryEncoding(num_tokens=dimension, name=f'one_hot_{c}')(inputs[c]))\n", - " else:\n", - " # embedding_size = int(math.floor(0.6 * dimension ** 0.25)), smaller model size for demo\n", - " embedding_size = 8\n", - " embeddings.append(Embedding(input_dim=dimension,\n", - " output_dim=embedding_size,\n", - " input_length=1,\n", - " name=f'embedding_{c}')(inputs[c]))\n", - "\n", - " x = Concatenate(name='embeddings_concat')(embeddings)\n", - " x = Flatten(name='embeddings_flatten')(x)\n", - " x = Concatenate(name='inputs_concat')([x] + one_hots + [inputs[i] for i in CONTINUOUS_COLUMNS])\n", - " x = BatchNormalization()(x)\n", - " x = Dense(64, activation='relu')(x)\n", - " x = BatchNormalization()(x)\n", - " x = Dense(64, activation='relu')(x)\n", - " x = BatchNormalization()(x)\n", - " x = Dense(64, activation='relu')(x)\n", - " x = BatchNormalization()(x)\n", - " x = Dense(32, activation='relu')(x)\n", - " output = Dense(1, activation='sigmoid', name='output')(x)\n", - " model = tf.keras.Model(inputs=[inputs[c] for c in ALL_COLUMNS], outputs=output)\n", - " if hvd.rank() == 0:\n", - " model.summary()\n", - "\n", - " opt = tf.keras.optimizers.Adam(learning_rate=args.learning_rate)\n", - " opt = hvd.DistributedOptimizer(opt)\n", - " model.compile(optimizer=opt, loss='binary_crossentropy', metrics=[tf.keras.metrics.AUC()])\n", - "\n", - " return model" - ] - }, - { - "cell_type": "markdown", - "id": "7dcc291d", - "metadata": {}, - "source": [ - "### Set train function\n", - "\n", - "The train_fn is the function that will execute across all Horovod workers(also say Spark executors in our case).\n", - "\n", - "We use `CUDA_VISIBLE_DEVICES` to control the GPU resources to avoid overlapping." - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "3bc96eb8", - "metadata": {}, - "outputs": [], - "source": [ - "def train_fn(dimensions, train_rows, val_rows, args):\n", - " # Make sure pyarrow is referenced before anything else to avoid segfault due to conflict\n", - " # with TensorFlow libraries. Use `pa` package reference to ensure it's loaded before\n", - " # functions like `deserialize_model` which are implemented at the top level.\n", - " # See https://jira.apache.org/jira/browse/ARROW-3346\n", - " pa\n", - "\n", - " import atexit\n", - " import horovod.tensorflow.keras as hvd\n", - " from horovod.spark.task import get_available_devices\n", - " import os\n", - " import tempfile\n", - " import tensorflow as tf\n", - " import tensorflow.keras.backend as K\n", - " import shutil\n", - "\n", - " gpus = get_available_devices()\n", - " if gpus:\n", - " os.environ['CUDA_VISIBLE_DEVICES'] = gpus[0]\n", - " if args.dataloader == NVTABULAR_DATALOADER:\n", - " os.environ['TF_MEMORY_ALLOCATION'] = '0.85'\n", - " from nvtabular.loader.tensorflow import KerasSequenceLoader\n", - "\n", - " # Horovod: initialize Horovod inside the trainer.\n", - " hvd.init()\n", - "\n", - " # Horovod: restore from checkpoint, use hvd.load_model under the hood.\n", - " model = build_model(dimensions, args)\n", - "\n", - " # Horovod: adjust learning rate based on number of processes.\n", - " scaled_lr = K.get_value(model.optimizer.lr) * hvd.size()\n", - " K.set_value(model.optimizer.lr, scaled_lr)\n", - "\n", - " # Horovod: print summary logs on the first worker.\n", - " verbose = 1 if hvd.rank() == 0 else 0\n", - "\n", - " callbacks = [\n", - " # Horovod: broadcast initial variable states from rank 0 to all other processes.\n", - " # This is necessary to ensure consistent initialization of all workers when\n", - " # training is started with random weights or restored from a checkpoint.\n", - " hvd.callbacks.BroadcastGlobalVariablesCallback(root_rank=0),\n", - "\n", - " # Horovod: average metrics among workers at the end of every epoch.\n", - " #\n", - " # Note: This callback must be in the list before the ReduceLROnPlateau,\n", - " # TensorBoard, or other metrics-based callbacks.\n", - " hvd.callbacks.MetricAverageCallback(),\n", - "\n", - " # Horovod: using `lr = 1.0 * hvd.size()` from the very beginning leads to worse final\n", - " # accuracy. Scale the learning rate `lr = 1.0` ---> `lr = 1.0 * hvd.size()` during\n", - " # the first five epochs. See https://arxiv.org/abs/1706.02677 for details.\n", - " hvd.callbacks.LearningRateWarmupCallback(initial_lr=scaled_lr, warmup_epochs=5, verbose=verbose),\n", - "\n", - " # Reduce LR if the metric is not improved for 10 epochs, and stop training\n", - " # if it has not improved for 20 epochs.\n", - " tf.keras.callbacks.ReduceLROnPlateau(monitor='val_auc', patience=10, verbose=verbose),\n", - " tf.keras.callbacks.EarlyStopping(monitor='val_auc', mode='min', patience=20, verbose=verbose),\n", - " tf.keras.callbacks.TerminateOnNaN(),\n", - "\n", - " # Log Tensorboard events.\n", - " tf.keras.callbacks.TensorBoard(log_dir=args.logs_dir, write_steps_per_second=True, update_freq=10)\n", - " ]\n", - "\n", - " # Horovod: save checkpoints only on the first worker to prevent other workers from corrupting them.\n", - " if hvd.rank() == 0:\n", - " ckpt_dir = tempfile.mkdtemp()\n", - " ckpt_file = os.path.join(ckpt_dir, 'checkpoint.h5')\n", - " atexit.register(lambda: shutil.rmtree(ckpt_dir))\n", - " callbacks.append(tf.keras.callbacks.ModelCheckpoint(\n", - " ckpt_file, monitor='val_auc', mode='min', save_best_only=True))\n", - "\n", - " if args.dataloader == PETASTORM_DATALOADER:\n", - " from petastorm import make_batch_reader\n", - " from petastorm.tf_utils import make_petastorm_dataset\n", - "\n", - " # Make Petastorm readers.\n", - " with make_batch_reader(f'{args.data_dir}/train',\n", - " num_epochs=None,\n", - " cur_shard=hvd.rank(),\n", - " shard_count=hvd.size(),\n", - " hdfs_driver='libhdfs') as train_reader:\n", - " with make_batch_reader(f'{args.data_dir}/val',\n", - " num_epochs=None,\n", - " cur_shard=hvd.rank(),\n", - " shard_count=hvd.size(),\n", - " hdfs_driver='libhdfs') as val_reader:\n", - " # Convert readers to tf.data.Dataset.\n", - " train_ds = make_petastorm_dataset(train_reader) \\\n", - " .unbatch() \\\n", - " .shuffle(10 * args.batch_size) \\\n", - " .batch(args.batch_size) \\\n", - " .map(lambda x: (tuple(getattr(x, c) for c in ALL_COLUMNS), x.clicked))\n", - "\n", - " val_ds = make_petastorm_dataset(val_reader) \\\n", - " .unbatch() \\\n", - " .batch(args.batch_size) \\\n", - " .map(lambda x: (tuple(getattr(x, c) for c in ALL_COLUMNS), x.clicked))\n", - "\n", - " history = model.fit(train_ds,\n", - " validation_data=val_ds,\n", - " steps_per_epoch=int(train_rows / args.batch_size / hvd.size()),\n", - " validation_steps=int(val_rows / args.batch_size / hvd.size()),\n", - " callbacks=callbacks,\n", - " verbose=verbose,\n", - " epochs=args.epochs)\n", - "\n", - " else:\n", - " import cupy\n", - "\n", - " def seed_fn():\n", - " \"\"\"\n", - " Generate consistent dataloader shuffle seeds across workers\n", - " Reseeds each worker's dataloader each epoch to get fresh a shuffle\n", - " that's consistent across workers.\n", - " \"\"\"\n", - " min_int, max_int = tf.int32.limits\n", - " max_rand = max_int // hvd.size()\n", - " # Generate a seed fragment on each worker\n", - " seed_fragment = cupy.random.randint(0, max_rand).get()\n", - " # Aggregate seed fragments from all Horovod workers\n", - " seed_tensor = tf.constant(seed_fragment)\n", - " reduced_seed = hvd.allreduce(seed_tensor, name=\"shuffle_seed\", op=hvd.Sum)\n", - " return reduced_seed % max_rand\n", - "\n", - " train_ds = KerasSequenceLoader(\n", - " f'{args.data_dir}/train',\n", - " batch_size=args.batch_size,\n", - " label_names=LABEL_COLUMNS,\n", - " cat_names=CATEGORICAL_COLUMNS,\n", - " cont_names=CONTINUOUS_COLUMNS,\n", - " engine=\"parquet\",\n", - " shuffle=True,\n", - " buffer_size=0.06, # how many batches to load at once\n", - " parts_per_chunk=1,\n", - " global_size=hvd.size(),\n", - " global_rank=hvd.rank(),\n", - " seed_fn=seed_fn)\n", - "\n", - " val_ds = KerasSequenceLoader(\n", - " f'{args.data_dir}/val',\n", - " batch_size=args.batch_size,\n", - " label_names=LABEL_COLUMNS,\n", - " cat_names=CATEGORICAL_COLUMNS,\n", - " cont_names=CONTINUOUS_COLUMNS,\n", - " engine=\"parquet\",\n", - " shuffle=False,\n", - " buffer_size=0.06, # how many batches to load at once\n", - " parts_per_chunk=1,\n", - " global_size=hvd.size(),\n", - " global_rank=hvd.rank())\n", - "\n", - " history = model.fit(train_ds,\n", - " validation_data=val_ds,\n", - " steps_per_epoch=int(train_rows / args.batch_size / hvd.size()),\n", - " validation_steps=int(val_rows / args.batch_size / hvd.size()),\n", - " callbacks=callbacks,\n", - " verbose=verbose,\n", - " epochs=args.epochs)\n", - "\n", - " if hvd.rank() == 0:\n", - " return history.history" - ] - }, - { - "cell_type": "markdown", - "id": "392ac264", - "metadata": {}, - "source": [ - "### Wrapper function to train\n", - "\n", - "Here we call the `horovod.spark.run` to start the training process in Horovod on Spark." - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "f9b8459d", - "metadata": {}, - "outputs": [], - "source": [ - "def train(dimensions, train_rows, val_rows, args):\n", - " # Horovod: run training.\n", - " history = horovod.spark.run(train_fn,\n", - " args=(dimensions, train_rows, val_rows, args),\n", - " num_proc=args.num_proc,\n", - " extra_mpi_args='-mca btl_tcp_if_include enp134s0f0 -x NCCL_IB_GID_INDEX=3',\n", - " stdout=sys.stdout,\n", - " stderr=sys.stderr,\n", - " verbose=2,\n", - " nics={},\n", - " prefix_output_with_timestamp=True)[0]\n", - "\n", - " best_val_loss = min(history['val_loss'])\n", - " print('Best Loss: %f' % best_val_loss)" - ] - }, - { - "cell_type": "markdown", - "id": "3e4ac6b8", - "metadata": {}, - "source": [ - "## Use NVTabular\n", - "\n", - "Here we set `--dataloader` to `nvtabular` to force NVTabular run." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "c2afc3a4", - "metadata": {}, - "outputs": [], - "source": [ - "def main():\n", - " parser = argparse.ArgumentParser(description='Criteo Spark Keras Training Example',\n", - " formatter_class=argparse.ArgumentDefaultsHelpFormatter)\n", - " parser.add_argument('--data-dir', default='file:///opt/data/criteo/parquet',\n", - " help='location of the transformed Criteo dataset in Parquet format')\n", - " parser.add_argument('--logs-dir', default='/opt/experiments/criteo', help='location of TensorFlow logs')\n", - " parser.add_argument('--dataloader', default=PETASTORM_DATALOADER,\n", - " choices=[PETASTORM_DATALOADER, NVTABULAR_DATALOADER],\n", - " help='dataloader to use')\n", - " parser.add_argument('--num-proc', type=int, default=1, help='number of worker processes for training')\n", - " parser.add_argument('--learning-rate', type=float, default=0.0001, help='initial learning rate')\n", - " parser.add_argument('--batch-size', type=int, default=64 * 1024, help='batch size')\n", - " parser.add_argument('--epochs', type=int, default=3, help='number of epochs to train')\n", - " parser.add_argument('--local-checkpoint-file', default='checkpoint', help='model checkpoint')\n", - " args = parser.parse_args(args=['--num-proc', '16', '--data-dir', 'file:///raid/spark-team/criteo/parquet', \n", - " '--dataloader', 'nvtabular', '--learning-rate', '0.001',\n", - " '--batch-size', '65535','--epochs', '1', '--logs-dir', 'tf_logs',\n", - " '--local-checkpoint-file', 'ckpt_file'])\n", - " \n", - "\n", - " dimensions = get_category_dimensions(spark, args.data_dir)\n", - "\n", - " train_df = spark.read.parquet(f'{args.data_dir}/train')\n", - " val_df = spark.read.parquet(f'{args.data_dir}/val')\n", - " test_df = spark.read.parquet(f'{args.data_dir}/test')\n", - " train_rows, val_rows, test_rows = train_df.count(), val_df.count(), test_df.count()\n", - " print('Training: %d' % train_rows)\n", - " print('Validation: %d' % val_rows)\n", - " print('Test: %d' % test_rows)\n", - "\n", - " train(dimensions, train_rows, val_rows, args)\n", - "\n", - " spark.stop()" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "a15573e3", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "21/09/06 09:03:44 WARN package: Truncated the string representation of a plan since it was too large. This behavior can be adjusted by setting 'spark.sql.debug.maxToStringFields'.\n", - " \r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'c0': '7912888',\n", - " 'c1': '33822',\n", - " 'c10': '582468',\n", - " 'c11': '245827',\n", - " 'c12': '10',\n", - " 'c13': '2208',\n", - " 'c14': '10666',\n", - " 'c15': '103',\n", - " 'c16': '3',\n", - " 'c17': '967',\n", - " 'c18': '14',\n", - " 'c19': '8165895',\n", - " 'c2': '17138',\n", - " 'c20': '2675939',\n", - " 'c21': '7156452',\n", - " 'c22': '302515',\n", - " 'c23': '12021',\n", - " 'c24': '96',\n", - " 'c25': '34',\n", - " 'c3': '7338',\n", - " 'c4': '20045',\n", - " 'c5': '3',\n", - " 'c6': '7104',\n", - " 'c7': '1381',\n", - " 'c8': '62',\n", - " 'c9': '5554113'}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - " \r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Training: 4195197692\n", - "Validation: 89137318\n", - "Test: 89137319\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Checking whether extension tensorflow was built with MPI.\n", - "Extension tensorflow was built with MPI.\n", - "mpirun --allow-run-as-root --tag-output -np 16 -H dgx2h0194-a1adff968d508e8d1142986f3e2c42dc:16 -bind-to none -map-by slot -mca pml ob1 -mca btl ^openib --timestamp-output -mca btl_tcp_if_include enp134s0f0 -x NCCL_IB_GID_INDEX=3 -x NCCL_DEBUG=INFO -mca plm_rsh_agent \"/home/ngc-auth-ldap-allxu/miniconda3/bin/python -m horovod.spark.driver.mpirun_rsh gAWVcAEAAAAAAAB9lCiMAmxvlF2UjAkxMjcuMC4wLjGUTWIShpRhjAdlbnA1M3MwlF2UjAwxMC4xNDguMzAuNTmUTWIShpRhjAdlbnA1OHMwlF2UjAwxMC4xNDguOTQuNTmUTWIShpRhjAdlbnA4OHMwlF2UjAwxMC4xNDkuMzAuMzSUTWIShpRhjAdlbnA5M3MwlF2UjAwxMC4xNDkuOTQuNTeUTWIShpRhjAplbnAxMzRzMGYwlF2UjAsxMC4xNTAuMzAuMpRNYhKGlGGMCGVucDE4NHMwlF2UjA0xMC4xNDguMTU4LjU5lE1iEoaUYYwIZW5wMTg5czCUXZSMDTEwLjE0OC4yMjIuNTmUTWIShpRhjAhlbnAyMjVzMJRdlIwNMTAuMTQ5LjE1OC41N5RNYhKGlGGMCGVucDIzMHMwlF2UjA0xMC4xNDkuMjIyLjU3lE1iEoaUYYwHZG9ja2VyMJRdlIwKMTcyLjE3LjAuMZRNYhKGlGF1Lg== gAWVAwMAAAAAAACMI2hvcm92b2QucnVubmVyLmNvbW1vbi51dGlsLnNldHRpbmdzlIwIU2V0dGluZ3OUk5QpgZR9lCiMCG51bV9wcm9jlEsQjAd2ZXJib3NllEsCjAhzc2hfcG9ydJROjBFzc2hfaWRlbnRpdHlfZmlsZZROjA5leHRyYV9tcGlfYXJnc5SMOS1tY2EgYnRsX3RjcF9pZl9pbmNsdWRlIGVucDEzNHMwZjAgLXggTkNDTF9JQl9HSURfSU5ERVg9M5SMCHRjcF9mbGFnlE6MDGJpbmRpbmdfYXJnc5ROjANrZXmUTowNc3RhcnRfdGltZW91dJSMImhvcm92b2QucnVubmVyLmNvbW1vbi51dGlsLnRpbWVvdXSUjAdUaW1lb3V0lJOUKYGUfZQojAhfdGltZW91dJRNWAKMC190aW1lb3V0X2F0lEdB2E128/YMbIwIX21lc3NhZ2WUWA4BAABUaW1lZCBvdXQgd2FpdGluZyBmb3Ige2FjdGl2aXR5fS4gUGxlYXNlIGNoZWNrIHRoYXQgeW91IGhhdmUgZW5vdWdoIHJlc291cmNlcyB0byBydW4gYWxsIEhvcm92b2QgcHJvY2Vzc2VzLiBFYWNoIEhvcm92b2QgcHJvY2VzcyBydW5zIGluIGEgU3BhcmsgdGFzay4gWW91IG1heSBuZWVkIHRvIGluY3JlYXNlIHRoZSBzdGFydF90aW1lb3V0IHBhcmFtZXRlciB0byBhIGxhcmdlciB2YWx1ZSBpZiB5b3VyIFNwYXJrIHJlc291cmNlcyBhcmUgYWxsb2NhdGVkIG9uLWRlbWFuZC6UdWKMD291dHB1dF9maWxlbmFtZZROjA1ydW5fZnVuY19tb2RllIiMBG5pY3OUfZSMB2VsYXN0aWOUiYwccHJlZml4X291dHB1dF93aXRoX3RpbWVzdGFtcJSIjAVob3N0c5SMLWRneDJoMDE5NC1hMWFkZmY5NjhkNTA4ZThkMTE0Mjk4NmYzZTJjNDJkYzoxNpR1Yi4=\" /home/ngc-auth-ldap-allxu/miniconda3/bin/python -m horovod.spark.task.mpirun_exec_fn gAWVcAEAAAAAAAB9lCiMAmxvlF2UjAkxMjcuMC4wLjGUTWIShpRhjAdlbnA1M3MwlF2UjAwxMC4xNDguMzAuNTmUTWIShpRhjAdlbnA1OHMwlF2UjAwxMC4xNDguOTQuNTmUTWIShpRhjAdlbnA4OHMwlF2UjAwxMC4xNDkuMzAuMzSUTWIShpRhjAdlbnA5M3MwlF2UjAwxMC4xNDkuOTQuNTeUTWIShpRhjAplbnAxMzRzMGYwlF2UjAsxMC4xNTAuMzAuMpRNYhKGlGGMCGVucDE4NHMwlF2UjA0xMC4xNDguMTU4LjU5lE1iEoaUYYwIZW5wMTg5czCUXZSMDTEwLjE0OC4yMjIuNTmUTWIShpRhjAhlbnAyMjVzMJRdlIwNMTAuMTQ5LjE1OC41N5RNYhKGlGGMCGVucDIzMHMwlF2UjA0xMC4xNDkuMjIyLjU3lE1iEoaUYYwHZG9ja2VyMJRdlIwKMTcyLjE3LjAuMZRNYhKGlGF1Lg== gAWVkwkAAAAAAACMI2hvcm92b2QucnVubmVyLmNvbW1vbi51dGlsLnNldHRpbmdzlIwIU2V0dGluZ3OUk5QpgZR9lCiMCG51bV9wcm9jlEsQjAd2ZXJib3NllEsCjAhzc2hfcG9ydJROjBFzc2hfaWRlbnRpdHlfZmlsZZROjA5leHRyYV9tcGlfYXJnc5RYxgYAAC1tY2EgYnRsX3RjcF9pZl9pbmNsdWRlIGVucDEzNHMwZjAgLXggTkNDTF9JQl9HSURfSU5ERVg9MyAteCBOQ0NMX0RFQlVHPUlORk8gLW1jYSBwbG1fcnNoX2FnZW50ICIvaG9tZS9uZ2MtYXV0aC1sZGFwLWFsbHh1L21pbmljb25kYTMvYmluL3B5dGhvbiAtbSBob3Jvdm9kLnNwYXJrLmRyaXZlci5tcGlydW5fcnNoIGdBV1ZjQUVBQUFBQUFBQjlsQ2lNQW14dmxGMlVqQWt4TWpjdU1DNHdMakdVVFdJU2hwUmhqQWRsYm5BMU0zTXdsRjJVakF3eE1DNHhORGd1TXpBdU5UbVVUV0lTaHBSaGpBZGxibkExT0hNd2xGMlVqQXd4TUM0eE5EZ3VPVFF1TlRtVVRXSVNocFJoakFkbGJuQTRPSE13bEYyVWpBd3hNQzR4TkRrdU16QXVNelNVVFdJU2hwUmhqQWRsYm5BNU0zTXdsRjJVakF3eE1DNHhORGt1T1RRdU5UZVVUV0lTaHBSaGpBcGxibkF4TXpSek1HWXdsRjJVakFzeE1DNHhOVEF1TXpBdU1wUk5ZaEtHbEdHTUNHVnVjREU0TkhNd2xGMlVqQTB4TUM0eE5EZ3VNVFU0TGpVNWxFMWlFb2FVWVl3SVpXNXdNVGc1Y3pDVVhaU01EVEV3TGpFME9DNHlNakl1TlRtVVRXSVNocFJoakFobGJuQXlNalZ6TUpSZGxJd05NVEF1TVRRNUxqRTFPQzQxTjVSTlloS0dsR0dNQ0dWdWNESXpNSE13bEYyVWpBMHhNQzR4TkRrdU1qSXlMalUzbEUxaUVvYVVZWXdIWkc5amEyVnlNSlJkbEl3S01UY3lMakUzTGpBdU1aUk5ZaEtHbEdGMUxnPT0gZ0FXVkF3TUFBQUFBQUFDTUkyaHZjbTkyYjJRdWNuVnVibVZ5TG1OdmJXMXZiaTUxZEdsc0xuTmxkSFJwYm1kemxJd0lVMlYwZEdsdVozT1VrNVFwZ1pSOWxDaU1DRzUxYlY5d2NtOWpsRXNRakFkMlpYSmliM05sbEVzQ2pBaHpjMmhmY0c5eWRKUk9qQkZ6YzJoZmFXUmxiblJwZEhsZlptbHNaWlJPakE1bGVIUnlZVjl0Y0dsZllYSm5jNVNNT1MxdFkyRWdZblJzWDNSamNGOXBabDlwYm1Oc2RXUmxJR1Z1Y0RFek5ITXdaakFnTFhnZ1RrTkRURjlKUWw5SFNVUmZTVTVFUlZnOU01U01DSFJqY0Y5bWJHRm5sRTZNREdKcGJtUnBibWRmWVhKbmM1Uk9qQU5yWlhtVVRvd05jM1JoY25SZmRHbHRaVzkxZEpTTUltaHZjbTkyYjJRdWNuVnVibVZ5TG1OdmJXMXZiaTUxZEdsc0xuUnBiV1Z2ZFhTVWpBZFVhVzFsYjNWMGxKT1VLWUdVZlpRb2pBaGZkR2x0Wlc5MWRKUk5XQUtNQzE5MGFXMWxiM1YwWDJGMGxFZEIyRTEyOC9ZTWJJd0lYMjFsYzNOaFoyV1VXQTRCQUFCVWFXMWxaQ0J2ZFhRZ2QyRnBkR2x1WnlCbWIzSWdlMkZqZEdsMmFYUjVmUzRnVUd4bFlYTmxJR05vWldOcklIUm9ZWFFnZVc5MUlHaGhkbVVnWlc1dmRXZG9JSEpsYzI5MWNtTmxjeUIwYnlCeWRXNGdZV3hzSUVodmNtOTJiMlFnY0hKdlkyVnpjMlZ6TGlCRllXTm9JRWh2Y205MmIyUWdjSEp2WTJWemN5QnlkVzV6SUdsdUlHRWdVM0JoY21zZ2RHRnpheTRnV1c5MUlHMWhlU0J1WldWa0lIUnZJR2x1WTNKbFlYTmxJSFJvWlNCemRHRnlkRjkwYVcxbGIzVjBJSEJoY21GdFpYUmxjaUIwYnlCaElHeGhjbWRsY2lCMllXeDFaU0JwWmlCNWIzVnlJRk53WVhKcklISmxjMjkxY21ObGN5QmhjbVVnWVd4c2IyTmhkR1ZrSUc5dUxXUmxiV0Z1WkM2VWRXS01EMjkxZEhCMWRGOW1hV3hsYm1GdFpaUk9qQTF5ZFc1ZlpuVnVZMTl0YjJSbGxJaU1CRzVwWTNPVWZaU01CMlZzWVhOMGFXT1VpWXdjY0hKbFptbDRYMjkxZEhCMWRGOTNhWFJvWDNScGJXVnpkR0Z0Y0pTSWpBVm9iM04wYzVTTUxXUm5lREpvTURFNU5DMWhNV0ZrWm1ZNU5qaGtOVEE0WlRoa01URTBNams0Tm1ZelpUSmpOREprWXpveE5wUjFZaTQ9IpSMCHRjcF9mbGFnlE6MDGJpbmRpbmdfYXJnc5ROjANrZXmUTowNc3RhcnRfdGltZW91dJSMImhvcm92b2QucnVubmVyLmNvbW1vbi51dGlsLnRpbWVvdXSUjAdUaW1lb3V0lJOUKYGUfZQojAhfdGltZW91dJRNWAKMC190aW1lb3V0X2F0lEdB2E128/YMbIwIX21lc3NhZ2WUWA4BAABUaW1lZCBvdXQgd2FpdGluZyBmb3Ige2FjdGl2aXR5fS4gUGxlYXNlIGNoZWNrIHRoYXQgeW91IGhhdmUgZW5vdWdoIHJlc291cmNlcyB0byBydW4gYWxsIEhvcm92b2QgcHJvY2Vzc2VzLiBFYWNoIEhvcm92b2QgcHJvY2VzcyBydW5zIGluIGEgU3BhcmsgdGFzay4gWW91IG1heSBuZWVkIHRvIGluY3JlYXNlIHRoZSBzdGFydF90aW1lb3V0IHBhcmFtZXRlciB0byBhIGxhcmdlciB2YWx1ZSBpZiB5b3VyIFNwYXJrIHJlc291cmNlcyBhcmUgYWxsb2NhdGVkIG9uLWRlbWFuZC6UdWKMD291dHB1dF9maWxlbmFtZZROjA1ydW5fZnVuY19tb2RllIiMBG5pY3OUfZSMB2VsYXN0aWOUiYwccHJlZml4X291dHB1dF93aXRoX3RpbWVzdGFtcJSIjAVob3N0c5SMLWRneDJoMDE5NC1hMWFkZmY5NjhkNTA4ZThkMTE0Mjk4NmYzZTJjNDJkYzoxNpR1Yi4=\n", - "Mon Sep 6 09:03:56 2021[1,2]:Changing cwd from /home/ngc-auth-ldap-allxu to /raid/spark-team/allen-dlrm/spark-3.1.2-bin-hadoop3.2/work/app-20210906090316-0000/1\n", - "Mon Sep 6 09:03:56 2021[1,4]:Changing cwd from /home/ngc-auth-ldap-allxu to /raid/spark-team/allen-dlrm/spark-3.1.2-bin-hadoop3.2/work/app-20210906090316-0000/1\n", - "Mon Sep 6 09:03:56 2021[1,0]:Changing cwd from /home/ngc-auth-ldap-allxu to /raid/spark-team/allen-dlrm/spark-3.1.2-bin-hadoop3.2/work/app-20210906090316-0000/1\n", - "Mon Sep 6 09:03:56 2021[1,6]:Changing cwd from /home/ngc-auth-ldap-allxu to /raid/spark-team/allen-dlrm/spark-3.1.2-bin-hadoop3.2/work/app-20210906090316-0000/1\n", - "Mon Sep 6 09:03:56 2021[1,1]:Changing cwd from /home/ngc-auth-ldap-allxu to /raid/spark-team/allen-dlrm/spark-3.1.2-bin-hadoop3.2/work/app-20210906090316-0000/1\n", - "Mon Sep 6 09:03:56 2021[1,5]:Changing cwd from /home/ngc-auth-ldap-allxu to /raid/spark-team/allen-dlrm/spark-3.1.2-bin-hadoop3.2/work/app-20210906090316-0000/1\n", - "Mon Sep 6 09:03:56 2021[1,3]:Changing cwd from /home/ngc-auth-ldap-allxu to /raid/spark-team/allen-dlrm/spark-3.1.2-bin-hadoop3.2/work/app-20210906090316-0000/1\n", - "Mon Sep 6 09:03:56 2021[1,10]:Changing cwd from /home/ngc-auth-ldap-allxu to /raid/spark-team/allen-dlrm/spark-3.1.2-bin-hadoop3.2/work/app-20210906090316-0000/1\n", - "Mon Sep 6 09:03:56 2021[1,7]:Changing cwd from /home/ngc-auth-ldap-allxu to /raid/spark-team/allen-dlrm/spark-3.1.2-bin-hadoop3.2/work/app-20210906090316-0000/1\n", - "Mon Sep 6 09:03:56 2021[1,15]:Changing cwd from /home/ngc-auth-ldap-allxu to /raid/spark-team/allen-dlrm/spark-3.1.2-bin-hadoop3.2/work/app-20210906090316-0000/1\n", - "Mon Sep 6 09:03:56 2021[1,11]:Changing cwd from /home/ngc-auth-ldap-allxu to /raid/spark-team/allen-dlrm/spark-3.1.2-bin-hadoop3.2/work/app-20210906090316-0000/1\n", - "Mon Sep 6 09:03:56 2021[1,13]:Changing cwd from /home/ngc-auth-ldap-allxu to /raid/spark-team/allen-dlrm/spark-3.1.2-bin-hadoop3.2/work/app-20210906090316-0000/1\n", - "Mon Sep 6 09:03:56 2021[1,14]:Changing cwd from /home/ngc-auth-ldap-allxu to /raid/spark-team/allen-dlrm/spark-3.1.2-bin-hadoop3.2/work/app-20210906090316-0000/1\n", - "Mon Sep 6 09:03:56 2021[1,8]:Changing cwd from /home/ngc-auth-ldap-allxu to /raid/spark-team/allen-dlrm/spark-3.1.2-bin-hadoop3.2/work/app-20210906090316-0000/1\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:03:56 2021[1,9]:Changing cwd from /home/ngc-auth-ldap-allxu to /raid/spark-team/allen-dlrm/spark-3.1.2-bin-hadoop3.2/work/app-20210906090316-0000/1\n", - "Mon Sep 6 09:03:56 2021[1,12]:Changing cwd from /home/ngc-auth-ldap-allxu to /raid/spark-team/allen-dlrm/spark-3.1.2-bin-hadoop3.2/work/app-20210906090316-0000/1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:16 2021[1,4]:2021-09-06 09:04:16.101560: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\n", - "Mon Sep 6 09:04:16 2021[1,4]:To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "Mon Sep 6 09:04:16 2021[1,2]:2021-09-06 09:04:16.101685: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\n", - "Mon Sep 6 09:04:16 2021[1,2]:To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "Mon Sep 6 09:04:16 2021[1,5]:2021-09-06 09:04:16.101662: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\n", - "Mon Sep 6 09:04:16 2021[1,5]:To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "Mon Sep 6 09:04:16 2021[1,13]:2021-09-06 09:04:16.101697: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\n", - "Mon Sep 6 09:04:16 2021[1,13]:To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "Mon Sep 6 09:04:16 2021[1,3]:2021-09-06 09:04:16.101749: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\n", - "Mon Sep 6 09:04:16 2021[1,3]:To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "Mon Sep 6 09:04:16 2021[1,1]:2021-09-06 09:04:16.102086: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\n", - "Mon Sep 6 09:04:16 2021[1,1]:To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "Mon Sep 6 09:04:16 2021[1,15]:2021-09-06 09:04:16.102170: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\n", - "Mon Sep 6 09:04:16 2021[1,15]:To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "Mon Sep 6 09:04:16 2021[1,8]:2021-09-06 09:04:16.102756: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\n", - "Mon Sep 6 09:04:16 2021[1,8]:To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "Mon Sep 6 09:04:16 2021[1,14]:2021-09-06 09:04:16.102705: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\n", - "Mon Sep 6 09:04:16 2021[1,14]:To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "Mon Sep 6 09:04:16 2021[1,7]:2021-09-06 09:04:16.103510: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\n", - "Mon Sep 6 09:04:16 2021[1,7]:To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "Mon Sep 6 09:04:16 2021[1,6]:2021-09-06 09:04:16.103663: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\n", - "Mon Sep 6 09:04:16 2021[1,6]:To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "Mon Sep 6 09:04:16 2021[1,12]:2021-09-06 09:04:16.103909: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\n", - "Mon Sep 6 09:04:16 2021[1,12]:To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "Mon Sep 6 09:04:16 2021[1,0]:2021-09-06 09:04:16.120233: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\n", - "Mon Sep 6 09:04:16 2021[1,0]:To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "Mon Sep 6 09:04:16 2021[1,10]:2021-09-06 09:04:16.120234: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\n", - "Mon Sep 6 09:04:16 2021[1,10]:To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "Mon Sep 6 09:04:16 2021[1,11]:2021-09-06 09:04:16.120458: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\n", - "Mon Sep 6 09:04:16 2021[1,11]:To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "Mon Sep 6 09:04:16 2021[1,9]:2021-09-06 09:04:16.121686: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\n", - "Mon Sep 6 09:04:16 2021[1,9]:To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "Mon Sep 6 09:04:18 2021[1,5]:2021-09-06 09:04:18.050711: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1504] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 27633 MB memory: -> device: 0, name: Tesla V100-SXM3-32GB-H, pci bus id: 0000:bc:00.0, compute capability: 7.0\n", - "Mon Sep 6 09:04:18 2021[1,2]:2021-09-06 09:04:18.111924: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1504] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 27633 MB memory: -> device: 0, name: Tesla V100-SXM3-32GB-H, pci bus id: 0000:b7:00.0, compute capability: 7.0\n", - "Mon Sep 6 09:04:18 2021[1,1]:2021-09-06 09:04:18.192508: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1504] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 27633 MB memory: -> device: 0, name: Tesla V100-SXM3-32GB-H, pci bus id: 0000:be:00.0, compute capability: 7.0\n", - "Mon Sep 6 09:04:18 2021[1,15]:2021-09-06 09:04:18.258547: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1504] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 27633 MB memory: -> device: 0, name: Tesla V100-SXM3-32GB-H, pci bus id: 0000:e5:00.0, compute capability: 7.0\n", - "Mon Sep 6 09:04:18 2021[1,9]:2021-09-06 09:04:18.290947: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1504] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 27633 MB memory: -> device: 0, name: Tesla V100-SXM3-32GB-H, pci bus id: 0000:59:00.0, compute capability: 7.0\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:18 2021[1,11]:2021-09-06 09:04:18.332073: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1504] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 27633 MB memory: -> device: 0, name: Tesla V100-SXM3-32GB-H, pci bus id: 0000:3b:00.0, compute capability: 7.0\n", - "Mon Sep 6 09:04:18 2021[1,4]:2021-09-06 09:04:18.333252: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1504] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 27633 MB memory: -> device: 0, name: Tesla V100-SXM3-32GB-H, pci bus id: 0000:e2:00.0, compute capability: 7.0\n", - "Mon Sep 6 09:04:18 2021[1,3]:2021-09-06 09:04:18.390624: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1504] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 27633 MB memory: -> device: 0, name: Tesla V100-SXM3-32GB-H, pci bus id: 0000:e7:00.0, compute capability: 7.0\n", - "Mon Sep 6 09:04:18 2021[1,7]:2021-09-06 09:04:18.391931: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1504] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 27633 MB memory: -> device: 0, name: Tesla V100-SXM3-32GB-H, pci bus id: 0000:b9:00.0, compute capability: 7.0\n", - "Mon Sep 6 09:04:18 2021[1,8]:2021-09-06 09:04:18.455155: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1504] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 27633 MB memory: -> device: 0, name: Tesla V100-SXM3-32GB-H, pci bus id: 0000:34:00.0, compute capability: 7.0\n", - "Mon Sep 6 09:04:18 2021[1,13]:2021-09-06 09:04:18.458402: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1504] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 27633 MB memory: -> device: 0, name: Tesla V100-SXM3-32GB-H, pci bus id: 0000:57:00.0, compute capability: 7.0\n", - "Mon Sep 6 09:04:18 2021[1,12]:2021-09-06 09:04:18.460130: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1504] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 27633 MB memory: -> device: 0, name: Tesla V100-SXM3-32GB-H, pci bus id: 0000:39:00.0, compute capability: 7.0\n", - "Mon Sep 6 09:04:18 2021[1,10]:2021-09-06 09:04:18.545289: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1504] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 27633 MB memory: -> device: 0, name: Tesla V100-SXM3-32GB-H, pci bus id: 0000:5e:00.0, compute capability: 7.0\n", - "Mon Sep 6 09:04:18 2021[1,14]:2021-09-06 09:04:18.545716: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1504] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 27633 MB memory: -> device: 0, name: Tesla V100-SXM3-32GB-H, pci bus id: 0000:e0:00.0, compute capability: 7.0\n", - "Mon Sep 6 09:04:18 2021[1,0]:2021-09-06 09:04:18.548694: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1504] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 27633 MB memory: -> device: 0, name: Tesla V100-SXM3-32GB-H, pci bus id: 0000:36:00.0, compute capability: 7.0\n", - "Mon Sep 6 09:04:18 2021[1,6]:2021-09-06 09:04:18.549196: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1504] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 27633 MB memory: -> device: 0, name: Tesla V100-SXM3-32GB-H, pci bus id: 0000:5c:00.0, compute capability: 7.0\n", - "Mon Sep 6 09:04:18 2021[1,5]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/horovod/_keras/callbacks.py:58: UserWarning: Some callbacks may not have access to the averaged metrics, see https://github.com/horovod/horovod/issues/2440\n", - "Mon Sep 6 09:04:18 2021[1,5]: warnings.warn(\n", - "Mon Sep 6 09:04:18 2021[1,5]:2021-09-06 09:04:18.864738: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 09:04:18 2021[1,5]:2021-09-06 09:04:18.864766: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 09:04:18 2021[1,5]:2021-09-06 09:04:18.864823: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1614] Profiler found 1 GPUs\n", - "Mon Sep 6 09:04:18 2021[1,2]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/horovod/_keras/callbacks.py:58: UserWarning: Some callbacks may not have access to the averaged metrics, see https://github.com/horovod/horovod/issues/2440\n", - "Mon Sep 6 09:04:18 2021[1,2]: warnings.warn(\n", - "Mon Sep 6 09:04:18 2021[1,2]:2021-09-06 09:04:18.915255: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 09:04:18 2021[1,2]:2021-09-06 09:04:18.915277: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 09:04:18 2021[1,2]:2021-09-06 09:04:18.915322: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1614] Profiler found 1 GPUs\n", - "Mon Sep 6 09:04:18 2021[1,1]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/horovod/_keras/callbacks.py:58: UserWarning: Some callbacks may not have access to the averaged metrics, see https://github.com/horovod/horovod/issues/2440\n", - "Mon Sep 6 09:04:18 2021[1,1]: warnings.warn(\n", - "Mon Sep 6 09:04:18 2021[1,1]:2021-09-06 09:04:18.936224: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 09:04:18 2021[1,1]:2021-09-06 09:04:18.936247: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 09:04:18 2021[1,1]:2021-09-06 09:04:18.936299: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1614] Profiler found 1 GPUs\n", - "Mon Sep 6 09:04:18 2021[1,15]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/horovod/_keras/callbacks.py:58: UserWarning: Some callbacks may not have access to the averaged metrics, see https://github.com/horovod/horovod/issues/2440\n", - "Mon Sep 6 09:04:18 2021[1,15]: warnings.warn(\n", - "Mon Sep 6 09:04:18 2021[1,15]:2021-09-06 09:04:18.987091: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 09:04:18 2021[1,15]:2021-09-06 09:04:18.987119: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 09:04:18 2021[1,15]:2021-09-06 09:04:18.987164: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1614] Profiler found 1 GPUs\n", - "Mon Sep 6 09:04:19 2021[1,9]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/horovod/_keras/callbacks.py:58: UserWarning: Some callbacks may not have access to the averaged metrics, see https://github.com/horovod/horovod/issues/2440\n", - "Mon Sep 6 09:04:19 2021[1,9]: warnings.warn(\n", - "Mon Sep 6 09:04:19 2021[1,9]:2021-09-06 09:04:19.009288: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 09:04:19 2021[1,9]:2021-09-06 09:04:19.009310: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 09:04:19 2021[1,9]:2021-09-06 09:04:19.009354: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1614] Profiler found 1 GPUs\n", - "Mon Sep 6 09:04:19 2021[1,4]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/horovod/_keras/callbacks.py:58: UserWarning: Some callbacks may not have access to the averaged metrics, see https://github.com/horovod/horovod/issues/2440\n", - "Mon Sep 6 09:04:19 2021[1,4]: warnings.warn(\n", - "Mon Sep 6 09:04:19 2021[1,4]:2021-09-06 09:04:19.063519: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 09:04:19 2021[1,4]:2021-09-06 09:04:19.063549: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 09:04:19 2021[1,4]:2021-09-06 09:04:19.063599: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1614] Profiler found 1 GPUs\n", - "Mon Sep 6 09:04:19 2021[1,11]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/horovod/_keras/callbacks.py:58: UserWarning: Some callbacks may not have access to the averaged metrics, see https://github.com/horovod/horovod/issues/2440\n", - "Mon Sep 6 09:04:19 2021[1,11]: warnings.warn(\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:19 2021[1,11]:2021-09-06 09:04:19.064564: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 09:04:19 2021[1,11]:2021-09-06 09:04:19.064584: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 09:04:19 2021[1,11]:2021-09-06 09:04:19.064626: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1614] Profiler found 1 GPUs\n", - "Mon Sep 6 09:04:19 2021[1,3]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/horovod/_keras/callbacks.py:58: UserWarning: Some callbacks may not have access to the averaged metrics, see https://github.com/horovod/horovod/issues/2440\n", - "Mon Sep 6 09:04:19 2021[1,3]: warnings.warn(\n", - "Mon Sep 6 09:04:19 2021[1,3]:2021-09-06 09:04:19.114647: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 09:04:19 2021[1,3]:2021-09-06 09:04:19.114675: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 09:04:19 2021[1,3]:2021-09-06 09:04:19.114721: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1614] Profiler found 1 GPUs\n", - "Mon Sep 6 09:04:19 2021[1,7]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/horovod/_keras/callbacks.py:58: UserWarning: Some callbacks may not have access to the averaged metrics, see https://github.com/horovod/horovod/issues/2440\n", - "Mon Sep 6 09:04:19 2021[1,7]: warnings.warn(\n", - "Mon Sep 6 09:04:19 2021[1,7]:2021-09-06 09:04:19.130694: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 09:04:19 2021[1,7]:2021-09-06 09:04:19.130715: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 09:04:19 2021[1,7]:2021-09-06 09:04:19.130764: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1614] Profiler found 1 GPUs\n", - "Mon Sep 6 09:04:19 2021[1,13]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/horovod/_keras/callbacks.py:58: UserWarning: Some callbacks may not have access to the averaged metrics, see https://github.com/horovod/horovod/issues/2440\n", - "Mon Sep 6 09:04:19 2021[1,13]: warnings.warn(\n", - "Mon Sep 6 09:04:19 2021[1,13]:2021-09-06 09:04:19.209150: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 09:04:19 2021[1,13]:2021-09-06 09:04:19.209178: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 09:04:19 2021[1,13]:2021-09-06 09:04:19.209230: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1614] Profiler found 1 GPUs\n", - "Mon Sep 6 09:04:19 2021[1,8]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/horovod/_keras/callbacks.py:58: UserWarning: Some callbacks may not have access to the averaged metrics, see https://github.com/horovod/horovod/issues/2440\n", - "Mon Sep 6 09:04:19 2021[1,8]: warnings.warn(\n", - "Mon Sep 6 09:04:19 2021[1,8]:2021-09-06 09:04:19.220618: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 09:04:19 2021[1,8]:2021-09-06 09:04:19.220639: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 09:04:19 2021[1,8]:2021-09-06 09:04:19.220688: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1614] Profiler found 1 GPUs\n", - "Mon Sep 6 09:04:19 2021[1,12]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/horovod/_keras/callbacks.py:58: UserWarning: Some callbacks may not have access to the averaged metrics, see https://github.com/horovod/horovod/issues/2440\n", - "Mon Sep 6 09:04:19 2021[1,12]: warnings.warn(\n", - "Mon Sep 6 09:04:19 2021[1,12]:2021-09-06 09:04:19.259405: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 09:04:19 2021[1,12]:2021-09-06 09:04:19.259428: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 09:04:19 2021[1,12]:2021-09-06 09:04:19.259476: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1614] Profiler found 1 GPUs\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:19 2021[1,0]:Model: \"model\"\n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:Layer (type) Output Shape Param # Connected to \n", - "Mon Sep 6 09:04:19 2021[1,0]:==================================================================================================\n", - "Mon Sep 6 09:04:19 2021[1,0]:c0 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:c1 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:c2 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:c3 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:c4 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:c6 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:c7 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:c8 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:c9 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:c10 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:c11 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:c12 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:c13 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:c14 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:c15 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:c17 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:c18 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:c19 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:c20 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:c21 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:c22 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:c23 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:c24 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:c25 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:embedding_c0 (Embedding) (None, 1, 8) 63303112 ['c0[0][0]'] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:embedding_c1 (Embedding) (None, 1, 8) 270584 ['c1[0][0]'] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:embedding_c2 (Embedding) (None, 1, 8) 137112 ['c2[0][0]'] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:embedding_c3 (Embedding) (None, 1, 8) 58712 ['c3[0][0]'] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:embedding_c4 (Embedding) (None, 1, 8) 160368 ['c4[0][0]'] \n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:embedding_c6 (Embedding) (None, 1, 8) 56840 ['c6[0][0]'] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:19 2021[1,0]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/horovod/_keras/callbacks.py:58: UserWarning: Some callbacks may not have access to the averaged metrics, see https://github.com/horovod/horovod/issues/2440\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:19 2021[1,0]:embedding_c7 (Embedding) (None, 1, 8) 11056 ['c7[0][0]'] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:19 2021[1,0]: warnings.warn(\n", - "Mon Sep 6 09:04:19 2021[1,0]:2021-09-06 09:04:19.311956: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:19 2021[1,0]:embedding_c8 (Embedding) (None, 1, 8) 504 ['c8[0][0]'] \n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:19 2021[1,0]:2021-09-06 09:04:19.311980: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:19 2021[1,0]:2021-09-06 09:04:19.312035: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1614] Profiler found 1 GPUs\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:19 2021[1,0]:embedding_c9 (Embedding) (None, 1, 8) 44432912 ['c9[0][0]'] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:embedding_c10 (Embedding) (None, 1, 8) 4659752 ['c10[0][0]'] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:embedding_c11 (Embedding) (None, 1, 8) 1966624 ['c11[0][0]'] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:embedding_c12 (Embedding) (None, 1, 8) 88 ['c12[0][0]'] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:embedding_c13 (Embedding) (None, 1, 8) 17672 ['c13[0][0]'] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:embedding_c14 (Embedding) (None, 1, 8) 85336 ['c14[0][0]'] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:embedding_c15 (Embedding) (None, 1, 8) 832 ['c15[0][0]'] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:embedding_c17 (Embedding) (None, 1, 8) 7744 ['c17[0][0]'] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:embedding_c18 (Embedding) (None, 1, 8) 120 ['c18[0][0]'] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:embedding_c19 (Embedding) (None, 1, 8) 65327168 ['c19[0][0]'] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:embedding_c20 (Embedding) (None, 1, 8) 21407520 ['c20[0][0]'] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:embedding_c21 (Embedding) (None, 1, 8) 57251624 ['c21[0][0]'] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:embedding_c22 (Embedding) (None, 1, 8) 2420128 ['c22[0][0]'] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:embedding_c23 (Embedding) (None, 1, 8) 96176 ['c23[0][0]'] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:embedding_c24 (Embedding) (None, 1, 8) 776 ['c24[0][0]'] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:embedding_c25 (Embedding) (None, 1, 8) 280 ['c25[0][0]'] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:embeddings_concat (Concatenate (None, 1, 192) 0 ['embedding_c0[0][0]', \n", - "Mon Sep 6 09:04:19 2021[1,0]:) 'embedding_c1[0][0]', \n", - "Mon Sep 6 09:04:19 2021[1,0]: 'embedding_c2[0][0]', \n", - "Mon Sep 6 09:04:19 2021[1,0]: 'embedding_c3[0][0]', \n", - "Mon Sep 6 09:04:19 2021[1,0]: 'embedding_c4[0][0]', \n", - "Mon Sep 6 09:04:19 2021[1,0]: 'embedding_c6[0][0]', \n", - "Mon Sep 6 09:04:19 2021[1,0]: 'embedding_c7[0][0]', \n", - "Mon Sep 6 09:04:19 2021[1,0]: 'embedding_c8[0][0]', \n", - "Mon Sep 6 09:04:19 2021[1,0]: 'embedding_c9[0][0]', \n", - "Mon Sep 6 09:04:19 2021[1,0]: 'embedding_c10[0][0]', \n", - "Mon Sep 6 09:04:19 2021[1,0]: 'embedding_c11[0][0]', \n", - "Mon Sep 6 09:04:19 2021[1,0]: 'embedding_c12[0][0]', \n", - "Mon Sep 6 09:04:19 2021[1,0]: 'embedding_c13[0][0]', \n", - "Mon Sep 6 09:04:19 2021[1,0]: 'embedding_c14[0][0]', \n", - "Mon Sep 6 09:04:19 2021[1,0]: 'embedding_c15[0][0]', \n", - "Mon Sep 6 09:04:19 2021[1,0]: 'embedding_c17[0][0]', \n", - "Mon Sep 6 09:04:19 2021[1,0]: 'embedding_c18[0][0]', \n", - "Mon Sep 6 09:04:19 2021[1,0]: 'embedding_c19[0][0]', \n", - "Mon Sep 6 09:04:19 2021[1,0]: 'embedding_c20[0][0]', \n", - "Mon Sep 6 09:04:19 2021[1,0]: 'embedding_c21[0][0]', \n", - "Mon Sep 6 09:04:19 2021[1,0]: 'embedding_c22[0][0]', \n", - "Mon Sep 6 09:04:19 2021[1,0]: 'embedding_c23[0][0]', \n", - "Mon Sep 6 09:04:19 2021[1,0]: 'embedding_c24[0][0]', \n", - "Mon Sep 6 09:04:19 2021[1,0]: 'embedding_c25[0][0]'] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:c5 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:c16 (InputLayer) [(None, 1)] 0 [] \n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:embeddings_flatten (Flatten) (None, 192) 0 ['embeddings_concat[0][0]'] \n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:19 2021[1,10]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/horovod/_keras/callbacks.py:58: UserWarning: Some callbacks may not have access to the averaged metrics, see https://github.com/horovod/horovod/issues/2440\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:one_hot_c5 (CategoryEncoding) (None, 4) 0 ['c5[0][0]'] \n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:19 2021[1,10]: warnings.warn(\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:one_hot_c16 (CategoryEncoding) (None, 4) 0 ['c16[0][0]'] \n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:19 2021[1,10]:2021-09-06 09:04:19.331721: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:19 2021[1,10]:2021-09-06 09:04:19.331743: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 09:04:19 2021[1,10]:2021-09-06 09:04:19.331792: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1614] Profiler found 1 GPUs\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:19 2021[1,0]:i0 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:i1 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:i2 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:i3 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:i4 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:i5 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:i6 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:i7 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:i8 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:i9 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:i10 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:i11 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:19 2021[1,6]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/horovod/_keras/callbacks.py:58: UserWarning: Some callbacks may not have access to the averaged metrics, see https://github.com/horovod/horovod/issues/2440\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:19 2021[1,0]:i12 (InputLayer) [(None, 1)] 0 [] \n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:19 2021[1,6]: warnings.warn(\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:19 2021[1,6]:2021-09-06 09:04:19.341050: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:19 2021[1,0]:inputs_concat (Concatenate) (None, 213) 0 ['embeddings_flatten[0][0]', \n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:19 2021[1,6]:2021-09-06 09:04:19.341072: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:19 2021[1,0]: 'one_hot_c5[0][0]', \n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:19 2021[1,6]:2021-09-06 09:04:19.341122: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1614] Profiler found 1 GPUs\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:19 2021[1,0]: 'one_hot_c16[0][0]', \n", - "Mon Sep 6 09:04:19 2021[1,0]: 'i0[0][0]', \n", - "Mon Sep 6 09:04:19 2021[1,0]: 'i1[0][0]', \n", - "Mon Sep 6 09:04:19 2021[1,0]: 'i2[0][0]', \n", - "Mon Sep 6 09:04:19 2021[1,0]: 'i3[0][0]', \n", - "Mon Sep 6 09:04:19 2021[1,0]: 'i4[0][0]', \n", - "Mon Sep 6 09:04:19 2021[1,0]: 'i5[0][0]', \n", - "Mon Sep 6 09:04:19 2021[1,0]: 'i6[0][0]', \n", - "Mon Sep 6 09:04:19 2021[1,0]: 'i7[0][0]', \n", - "Mon Sep 6 09:04:19 2021[1,0]: 'i8[0][0]', \n", - "Mon Sep 6 09:04:19 2021[1,0]: 'i9[0][0]', \n", - "Mon Sep 6 09:04:19 2021[1,0]: 'i10[0][0]', \n", - "Mon Sep 6 09:04:19 2021[1,0]: 'i11[0][0]', \n", - "Mon Sep 6 09:04:19 2021[1,0]: 'i12[0][0]'] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:batch_normalization (BatchNorm (None, 213) 852 ['inputs_concat[0][0]'] \n", - "Mon Sep 6 09:04:19 2021[1,0]:alization) \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:dense (Dense) (None, 64) 13696 ['batch_normalization[0][0]'] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:batch_normalization_1 (BatchNo (None, 64) 256 ['dense[0][0]'] \n", - "Mon Sep 6 09:04:19 2021[1,0]:rmalization) \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:dense_1 (Dense) (None, 64) 4160 ['batch_normalization_1[0][0]'] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:batch_normalization_2 (BatchNo (None, 64) 256 ['dense_1[0][0]'] \n", - "Mon Sep 6 09:04:19 2021[1,0]:rmalization) \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:dense_2 (Dense) (None, 64) 4160 ['batch_normalization_2[0][0]'] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:batch_normalization_3 (BatchNo (None, 64) 256 ['dense_2[0][0]'] \n", - "Mon Sep 6 09:04:19 2021[1,0]:rmalization) \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:dense_3 (Dense) (None, 32) 2080 ['batch_normalization_3[0][0]'] \n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 09:04:19 2021[1,0]:output (Dense) (None, 1) 33 ['dense_3[0][0]'] \n", - "Mon Sep 6 09:04:19 2021[1,0]:==================================================================================================\n", - "Mon Sep 6 09:04:19 2021[1,0]:Total params: 261,698,789\n", - "Mon Sep 6 09:04:19 2021[1,0]:Trainable params: 261,697,979\n", - "Mon Sep 6 09:04:19 2021[1,0]:Non-trainable params: 810\n", - "Mon Sep 6 09:04:19 2021[1,0]:__________________________________________________________________________________________________\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:19 2021[1,14]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/horovod/_keras/callbacks.py:58: UserWarning: Some callbacks may not have access to the averaged metrics, see https://github.com/horovod/horovod/issues/2440\n", - "Mon Sep 6 09:04:19 2021[1,14]: warnings.warn(\n", - "Mon Sep 6 09:04:19 2021[1,14]:2021-09-06 09:04:19.369461: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 09:04:19 2021[1,14]:2021-09-06 09:04:19.369483: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 09:04:19 2021[1,14]:2021-09-06 09:04:19.369533: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1614] Profiler found 1 GPUs\n", - "Mon Sep 6 09:04:19 2021[1,2]:2021-09-06 09:04:19.714971: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 09:04:19 2021[1,2]:2021-09-06 09:04:19.719251: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 09:04:19 2021[1,5]:2021-09-06 09:04:19.726330: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 09:04:19 2021[1,5]:2021-09-06 09:04:19.730619: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 09:04:19 2021[1,1]:2021-09-06 09:04:19.849382: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 09:04:19 2021[1,1]:2021-09-06 09:04:19.852162: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 09:04:19 2021[1,9]:2021-09-06 09:04:19.976160: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 09:04:19 2021[1,9]:2021-09-06 09:04:19.979006: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 09:04:20 2021[1,15]:2021-09-06 09:04:20.045149: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 09:04:20 2021[1,15]:2021-09-06 09:04:20.047988: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 09:04:20 2021[1,11]:2021-09-06 09:04:20.289338: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 09:04:20 2021[1,11]:2021-09-06 09:04:20.292348: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 09:04:20 2021[1,4]:2021-09-06 09:04:20.312786: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 09:04:20 2021[1,4]:2021-09-06 09:04:20.315716: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 09:04:20 2021[1,3]:2021-09-06 09:04:20.391964: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 09:04:20 2021[1,3]:2021-09-06 09:04:20.394729: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 09:04:20 2021[1,12]:2021-09-06 09:04:20.510335: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 09:04:20 2021[1,12]:2021-09-06 09:04:20.514644: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 09:04:20 2021[1,8]:2021-09-06 09:04:20.573990: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 09:04:20 2021[1,7]:2021-09-06 09:04:20.577094: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 09:04:20 2021[1,8]:2021-09-06 09:04:20.583448: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 09:04:20 2021[1,7]:2021-09-06 09:04:20.591334: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 09:04:20 2021[1,13]:2021-09-06 09:04:20.631241: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 09:04:20 2021[1,10]:2021-09-06 09:04:20.639145: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 09:04:20 2021[1,13]:2021-09-06 09:04:20.639919: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 09:04:20 2021[1,10]:2021-09-06 09:04:20.641938: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 09:04:20 2021[1,0]:2021-09-06 09:04:20.642622: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 09:04:20 2021[1,0]:2021-09-06 09:04:20.645310: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 09:04:20 2021[1,14]:2021-09-06 09:04:20.672134: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 09:04:20 2021[1,14]:2021-09-06 09:04:20.676577: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 09:04:20 2021[1,6]:2021-09-06 09:04:20.851373: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 09:04:20 2021[1,6]:2021-09-06 09:04:20.855704: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:45 2021[1,0]:dgx2h0194:79712:80584 [0] NCCL INFO Bootstrap : Using enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 09:04:45 2021[1,0]:dgx2h0194:79712:80584 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation\n", - "Mon Sep 6 09:04:46 2021[1,0]:dgx2h0194:79712:80584 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [4]mlx5_4:1/RoCE [5]mlx5_5:1/RoCE [6]mlx5_6:1/RoCE [7]mlx5_7:1/RoCE [8]mlx5_8:1/RoCE [9]mlx5_9:1/RoCE ; OOB enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 09:04:46 2021[1,0]:dgx2h0194:79712:80584 [0] NCCL INFO Using network IB\n", - "Mon Sep 6 09:04:46 2021[1,0]:NCCL version 2.10.3+cuda11.0\n", - "Mon Sep 6 09:04:46 2021[1,10]:dgx2h0194:79743:80602 [0] NCCL INFO Bootstrap : Using enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 09:04:46 2021[1,10]:dgx2h0194:79743:80602 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation\n", - "Mon Sep 6 09:04:46 2021[1,13]:dgx2h0194:79746:80607 [0] NCCL INFO Bootstrap : Using enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 09:04:46 2021[1,13]:dgx2h0194:79746:80607 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation\n", - "Mon Sep 6 09:04:46 2021[1,6]:dgx2h0194:79739:80608 [0] NCCL INFO Bootstrap : Using enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 09:04:46 2021[1,6]:dgx2h0194:79739:80608 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation\n", - "Mon Sep 6 09:04:46 2021[1,4]:dgx2h0194:79732:80427 [0] NCCL INFO Bootstrap : Using enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 09:04:46 2021[1,4]:dgx2h0194:79732:80427 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation\n", - "Mon Sep 6 09:04:46 2021[1,10]:dgx2h0194:79743:80602 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [4]mlx5_4:1/RoCE [5]mlx5_5:1/RoCE [6]mlx5_6:1/RoCE [7]mlx5_7:1/RoCE [8]mlx5_8:1/RoCE [9]mlx5_9:1/RoCE ; OOB enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 09:04:46 2021[1,10]:dgx2h0194:79743:80602 [0] NCCL INFO Using network IB\n", - "Mon Sep 6 09:04:46 2021[1,2]:dgx2h0194:79722:80271 [0] NCCL INFO Bootstrap : Using enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 09:04:46 2021[1,2]:dgx2h0194:79722:80271 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation\n", - "Mon Sep 6 09:04:46 2021[1,1]:dgx2h0194:79717:80738 [0] NCCL INFO Bootstrap : Using enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 09:04:46 2021[1,13]:dgx2h0194:79746:80607 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [4]mlx5_4:1/RoCE [5]mlx5_5:1/RoCE [6]mlx5_6:1/RoCE [7]mlx5_7:1/RoCE [8]mlx5_8:1/RoCE [9]mlx5_9:1/RoCE ; OOB enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 09:04:46 2021[1,13]:dgx2h0194:79746:80607 [0] NCCL INFO Using network IB\n", - "Mon Sep 6 09:04:46 2021[1,1]:dgx2h0194:79717:80738 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation\n", - "Mon Sep 6 09:04:46 2021[1,15]:dgx2h0194:79748:80424 [0] NCCL INFO Bootstrap : Using enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 09:04:46 2021[1,15]:dgx2h0194:79748:80424 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation\n", - "Mon Sep 6 09:04:46 2021[1,5]:dgx2h0194:79738:80437 [0] NCCL INFO Bootstrap : Using enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 09:04:46 2021[1,5]:dgx2h0194:79738:80437 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation\n", - "Mon Sep 6 09:04:46 2021[1,6]:dgx2h0194:79739:80608 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [4]mlx5_4:1/RoCE [5]mlx5_5:1/RoCE [6]mlx5_6:1/RoCE [7]mlx5_7:1/RoCE [8]mlx5_8:1/RoCE [9]mlx5_9:1/RoCE ; OOB enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 09:04:46 2021[1,6]:dgx2h0194:79739:80608 [0] NCCL INFO Using network IB\n", - "Mon Sep 6 09:04:46 2021[1,4]:dgx2h0194:79732:80427 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [4]mlx5_4:1/RoCE [5]mlx5_5:1/RoCE [6]mlx5_6:1/RoCE [7]mlx5_7:1/RoCE [8]mlx5_8:1/RoCE [9]mlx5_9:1/RoCE ; OOB enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 09:04:46 2021[1,4]:dgx2h0194:79732:80427 [0] NCCL INFO Using network IB\n", - "Mon Sep 6 09:04:46 2021[1,7]:dgx2h0194:79740:80605 [0] NCCL INFO Bootstrap : Using enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 09:04:46 2021[1,7]:dgx2h0194:79740:80605 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation\n", - "Mon Sep 6 09:04:46 2021[1,2]:dgx2h0194:79722:80271 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [4]mlx5_4:1/RoCE [5]mlx5_5:1/RoCE [6]mlx5_6:1/RoCE [7]mlx5_7:1/RoCE [8]mlx5_8:1/RoCE [9]mlx5_9:1/RoCE ; OOB enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 09:04:46 2021[1,2]:dgx2h0194:79722:80271 [0] NCCL INFO Using network IB\n", - "Mon Sep 6 09:04:46 2021[1,1]:dgx2h0194:79717:80738 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [4]mlx5_4:1/RoCE [5]mlx5_5:1/RoCE [6]mlx5_6:1/RoCE [7]mlx5_7:1/RoCE [8]mlx5_8:1/RoCE [9]mlx5_9:1/RoCE ; OOB enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 09:04:46 2021[1,1]:dgx2h0194:79717:80738 [0] NCCL INFO Using network IB\n", - "Mon Sep 6 09:04:46 2021[1,15]:dgx2h0194:79748:80424 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [4]mlx5_4:1/RoCE [5]mlx5_5:1/RoCE [6]mlx5_6:1/RoCE [7]mlx5_7:1/RoCE [8]mlx5_8:1/RoCE [9]mlx5_9:1/RoCE ; OOB enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 09:04:46 2021[1,15]:dgx2h0194:79748:80424 [0] NCCL INFO Using network IB\n", - "Mon Sep 6 09:04:46 2021[1,9]:dgx2h0194:79742:80744 [0] NCCL INFO Bootstrap : Using enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 09:04:46 2021[1,9]:dgx2h0194:79742:80744 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation\n", - "Mon Sep 6 09:04:46 2021[1,5]:dgx2h0194:79738:80437 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [4]mlx5_4:1/RoCE [5]mlx5_5:1/RoCE [6]mlx5_6:1/RoCE [7]mlx5_7:1/RoCE [8]mlx5_8:1/RoCE [9]mlx5_9:1/RoCE ; OOB enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 09:04:46 2021[1,5]:dgx2h0194:79738:80437 [0] NCCL INFO Using network IB\n", - "Mon Sep 6 09:04:46 2021[1,7]:dgx2h0194:79740:80605 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [4]mlx5_4:1/RoCE [5]mlx5_5:1/RoCE [6]mlx5_6:1/RoCE [7]mlx5_7:1/RoCE [8]mlx5_8:1/RoCE [9]mlx5_9:1/RoCE ; OOB enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 09:04:46 2021[1,7]:dgx2h0194:79740:80605 [0] NCCL INFO Using network IB\n", - "Mon Sep 6 09:04:46 2021[1,8]:dgx2h0194:79741:80747 [0] NCCL INFO Bootstrap : Using enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 09:04:46 2021[1,8]:dgx2h0194:79741:80747 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation\n", - "Mon Sep 6 09:04:46 2021[1,9]:dgx2h0194:79742:80744 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [4]mlx5_4:1/RoCE [5]mlx5_5:1/RoCE [6]mlx5_6:1/RoCE [7]mlx5_7:1/RoCE [8]mlx5_8:1/RoCE [9]mlx5_9:1/RoCE ; OOB enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 09:04:46 2021[1,9]:dgx2h0194:79742:80744 [0] NCCL INFO Using network IB\n", - "Mon Sep 6 09:04:46 2021[1,8]:dgx2h0194:79741:80747 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [4]mlx5_4:1/RoCE [5]mlx5_5:1/RoCE [6]mlx5_6:1/RoCE [7]mlx5_7:1/RoCE [8]mlx5_8:1/RoCE [9]mlx5_9:1/RoCE ; OOB enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 09:04:46 2021[1,8]:dgx2h0194:79741:80747 [0] NCCL INFO Using network IB\n", - "Mon Sep 6 09:04:46 2021[1,3]:dgx2h0194:79727:80587 [0] NCCL INFO Bootstrap : Using enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 09:04:46 2021[1,3]:dgx2h0194:79727:80587 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation\n", - "Mon Sep 6 09:04:46 2021[1,11]:dgx2h0194:79744:80434 [0] NCCL INFO Bootstrap : Using enp53s0:10.148.30.59<0>\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:46 2021[1,11]:dgx2h0194:79744:80434 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation\n", - "Mon Sep 6 09:04:46 2021[1,12]:dgx2h0194:79745:80741 [0] NCCL INFO Bootstrap : Using enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 09:04:46 2021[1,12]:dgx2h0194:79745:80741 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation\n", - "Mon Sep 6 09:04:46 2021[1,3]:dgx2h0194:79727:80587 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [4]mlx5_4:1/RoCE [5]mlx5_5:1/RoCE [6]mlx5_6:1/RoCE [7]mlx5_7:1/RoCE [8]mlx5_8:1/RoCE [9]mlx5_9:1/RoCE ; OOB enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 09:04:46 2021[1,3]:dgx2h0194:79727:80587 [0] NCCL INFO Using network IB\n", - "Mon Sep 6 09:04:46 2021[1,11]:dgx2h0194:79744:80434 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [4]mlx5_4:1/RoCE [5]mlx5_5:1/RoCE [6]mlx5_6:1/RoCE [7]mlx5_7:1/RoCE [8]mlx5_8:1/RoCE [9]mlx5_9:1/RoCE ; OOB enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 09:04:46 2021[1,11]:dgx2h0194:79744:80434 [0] NCCL INFO Using network IB\n", - "Mon Sep 6 09:04:46 2021[1,14]:dgx2h0194:79747:80615 [0] NCCL INFO Bootstrap : Using enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 09:04:46 2021[1,14]:dgx2h0194:79747:80615 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation\n", - "Mon Sep 6 09:04:46 2021[1,12]:dgx2h0194:79745:80741 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [4]mlx5_4:1/RoCE [5]mlx5_5:1/RoCE [6]mlx5_6:1/RoCE [7]mlx5_7:1/RoCE [8]mlx5_8:1/RoCE [9]mlx5_9:1/RoCE ; OOB enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 09:04:46 2021[1,12]:dgx2h0194:79745:80741 [0] NCCL INFO Using network IB\n", - "Mon Sep 6 09:04:46 2021[1,14]:dgx2h0194:79747:80615 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [4]mlx5_4:1/RoCE [5]mlx5_5:1/RoCE [6]mlx5_6:1/RoCE [7]mlx5_7:1/RoCE [8]mlx5_8:1/RoCE [9]mlx5_9:1/RoCE ; OOB enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 09:04:46 2021[1,14]:dgx2h0194:79747:80615 [0] NCCL INFO Using network IB\n", - "Mon Sep 6 09:04:46 2021[1,12]:dgx2h0194:79745:80741 [0] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.\n", - "Mon Sep 6 09:04:46 2021[1,8]:dgx2h0194:79741:80747 [0] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.\n", - "Mon Sep 6 09:04:46 2021[1,0]:dgx2h0194:79712:80584 [0] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.\n", - "Mon Sep 6 09:04:46 2021[1,2]:dgx2h0194:79722:80271 [0] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.\n", - "Mon Sep 6 09:04:46 2021[1,3]:dgx2h0194:79727:80587 [0] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.\n", - "Mon Sep 6 09:04:46 2021[1,4]:dgx2h0194:79732:80427 [0] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.\n", - "Mon Sep 6 09:04:46 2021[1,13]:dgx2h0194:79746:80607 [0] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.\n", - "Mon Sep 6 09:04:46 2021[1,5]:dgx2h0194:79738:80437 [0] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.\n", - "Mon Sep 6 09:04:46 2021[1,7]:dgx2h0194:79740:80605 [0] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.\n", - "Mon Sep 6 09:04:46 2021[1,15]:dgx2h0194:79748:80424 [0] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.\n", - "Mon Sep 6 09:04:46 2021[1,6]:dgx2h0194:79739:80608 [0] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.\n", - "Mon Sep 6 09:04:46 2021[1,10]:dgx2h0194:79743:80602 [0] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.\n", - "Mon Sep 6 09:04:46 2021[1,14]:dgx2h0194:79747:80615 [0] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.\n", - "Mon Sep 6 09:04:46 2021[1,9]:dgx2h0194:79742:80744 [0] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.\n", - "Mon Sep 6 09:04:46 2021[1,11]:dgx2h0194:79744:80434 [0] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.\n", - "Mon Sep 6 09:04:46 2021[1,1]:dgx2h0194:79717:80738 [0] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:57 2021[1,13]:dgx2h0194:79746:80607 [0] NCCL INFO Trees [0] 1/-1/-1->13->9 [1] 1/-1/-1->13->9 [2] 1/-1/-1->13->9 [3] 1/-1/-1->13->9 [4] 1/-1/-1->13->9 [5] 1/-1/-1->13->9 [6] 1/-1/-1->13->9 [7] 1/-1/-1->13->9 [8] 1/-1/-1->13->9 [9] 1/-1/-1->13->9 [10] 1/-1/-1->13->9 [11] 1/-1/-1->13->9\n", - "Mon Sep 6 09:04:57 2021[1,13]:dgx2h0194:79746:80607 [0] NCCL INFO Setting affinity for GPU 4 to ff,ffff0000,00ffffff\n", - "Mon Sep 6 09:04:57 2021[1,12]:dgx2h0194:79745:80741 [0] NCCL INFO Trees [0] 6/-1/-1->12->11 [1] 6/-1/-1->12->11 [2] 6/-1/-1->12->11 [3] 6/-1/-1->12->11 [4] 6/-1/-1->12->11 [5] 6/-1/-1->12->11 [6] 6/-1/-1->12->11 [7] 6/-1/-1->12->11 [8] 6/-1/-1->12->11 [9] 6/-1/-1->12->11 [10] 6/-1/-1->12->11 [11] 6/-1/-1->12->11\n", - "Mon Sep 6 09:04:57 2021[1,12]:dgx2h0194:79745:80741 [0] NCCL INFO Setting affinity for GPU 2 to ff,ffff0000,00ffffff\n", - "Mon Sep 6 09:04:57 2021[1,15]:dgx2h0194:79748:80424 [0] NCCL INFO Trees [0] 4/-1/-1->15->3 [1] 4/-1/-1->15->3 [2] 4/-1/-1->15->3 [3] 4/-1/-1->15->3 [4] 4/-1/-1->15->3 [5] 4/-1/-1->15->3 [6] 4/-1/-1->15->3 [7] 4/-1/-1->15->3 [8] 4/-1/-1->15->3 [9] 4/-1/-1->15->3 [10] 4/-1/-1->15->3 [11] 4/-1/-1->15->3\n", - "Mon Sep 6 09:04:57 2021[1,15]:dgx2h0194:79748:80424 [0] NCCL INFO Setting affinity for GPU 14 to ffffff00,0000ffff,ff000000\n", - "Mon Sep 6 09:04:57 2021[1,14]:dgx2h0194:79747:80615 [0] NCCL INFO Trees [0] -1/-1/-1->14->4 [1] -1/-1/-1->14->4 [2] -1/-1/-1->14->4 [3] -1/-1/-1->14->4 [4] -1/-1/-1->14->4 [5] -1/-1/-1->14->4 [6] -1/-1/-1->14->4 [7] -1/-1/-1->14->4 [8] -1/-1/-1->14->4 [9] -1/-1/-1->14->4 [10] -1/-1/-1->14->4 [11] -1/-1/-1->14->4\n", - "Mon Sep 6 09:04:57 2021[1,14]:dgx2h0194:79747:80615 [0] NCCL INFO Setting affinity for GPU 12 to ffffff00,0000ffff,ff000000\n", - "Mon Sep 6 09:04:57 2021[1,0]:dgx2h0194:79712:80584 [0] NCCL INFO Channel 00/12 : 0 8 11 12 6 10 9 13 1 5 2 7 3 15 4 14\n", - "Mon Sep 6 09:04:57 2021[1,0]:dgx2h0194:79712:80584 [0] NCCL INFO Channel 01/12 : 0 8 11 12 6 10 9 13 1 5 2 7 3 15 4 14\n", - "Mon Sep 6 09:04:57 2021[1,0]:dgx2h0194:79712:80584 [0] NCCL INFO Channel 02/12 : 0 8 11 12 6 10 9 13 1 5 2 7 3 15 4 14\n", - "Mon Sep 6 09:04:57 2021[1,0]:dgx2h0194:79712:80584 [0] NCCL INFO Channel 03/12 : 0 8 11 12 6 10 9 13 1 5 2 7 3 15 4 14\n", - "Mon Sep 6 09:04:57 2021[1,0]:dgx2h0194:79712:80584 [0] NCCL INFO Channel 04/12 : 0 8 11 12 6 10 9 13 1 5 2 7 3 15 4 14\n", - "Mon Sep 6 09:04:57 2021[1,0]:dgx2h0194:79712:80584 [0] NCCL INFO Channel 05/12 : 0 8 11 12 6 10 9 13 1 5 2 7 3 15 4 14\n", - "Mon Sep 6 09:04:57 2021[1,0]:dgx2h0194:79712:80584 [0] NCCL INFO Channel 06/12 : 0 8 11 12 6 10 9 13 1 5 2 7 3 15 4 14\n", - "Mon Sep 6 09:04:57 2021[1,0]:dgx2h0194:79712:80584 [0] NCCL INFO Channel 07/12 : 0 8 11 12 6 10 9 13 1 5 2 7 3 15 4 14\n", - "Mon Sep 6 09:04:57 2021[1,0]:dgx2h0194:79712:80584 [0] NCCL INFO Channel 08/12 : 0 8 11 12 6 10 9 13 1 5 2 7 3 15 4 14\n", - "Mon Sep 6 09:04:57 2021[1,1]:dgx2h0194:79717:80738 [0] NCCL INFO Trees [0] 5/-1/-1->1->13 [1] 5/-1/-1->1->13 [2] 5/-1/-1->1->13 [3] 5/-1/-1->1->13 [4] 5/-1/-1->1->13 [5] 5/-1/-1->1->13 [6] 5/-1/-1->1->13 [7] 5/-1/-1->1->13 [8] 5/-1/-1->1->13 [9] 5/-1/-1->1->13 [10] 5/-1/-1->1->13 [11] 5/-1/-1->1->13\n", - "Mon Sep 6 09:04:57 2021[1,1]:dgx2h0194:79717:80738 [0] NCCL INFO Setting affinity for GPU 11 to ffffff00,0000ffff,ff000000\n", - "Mon Sep 6 09:04:57 2021[1,0]:dgx2h0194:79712:80584 [0] NCCL INFO Channel 09/12 : 0 8 11 12 6 10 9 13 1 5 2 7 3 15 4 14\n", - "Mon Sep 6 09:04:57 2021[1,0]:dgx2h0194:79712:80584 [0] NCCL INFO Channel 10/12 : 0 8 11 12 6 10 9 13 1 5 2 7 3 15 4 14\n", - "Mon Sep 6 09:04:57 2021[1,0]:dgx2h0194:79712:80584 [0] NCCL INFO Channel 11/12 : 0 8 11 12 6 10 9 13 1 5 2 7 3 15 4 14\n", - "Mon Sep 6 09:04:57 2021[1,0]:dgx2h0194:79712:80584 [0] NCCL INFO Trees [0] 8/-1/-1->0->-1 [1] 8/-1/-1->0->-1 [2] 8/-1/-1->0->-1 [3] 8/-1/-1->0->-1 [4] 8/-1/-1->0->-1 [5] 8/-1/-1->0->-1 [6] 8/-1/-1->0->-1 [7] 8/-1/-1->0->-1 [8] 8/-1/-1->0->-1 [9] 8/-1/-1->0->-1 [10] 8/-1/-1->0->-1 [11] 8/-1/-1->0->-1\n", - "Mon Sep 6 09:04:57 2021[1,11]:dgx2h0194:79744:80434 [0] NCCL INFO Trees [0] 12/-1/-1->11->8 [1] 12/-1/-1->11->8 [2] 12/-1/-1->11->8 [3] 12/-1/-1->11->8 [4] 12/-1/-1->11->8 [5] 12/-1/-1->11->8 [6] 12/-1/-1->11->8 [7] 12/-1/-1->11->8 [8] 12/-1/-1->11->8 [9] 12/-1/-1->11->8 [10] 12/-1/-1->11->8 [11] 12/-1/-1->11->8\n", - "Mon Sep 6 09:04:57 2021[1,11]:dgx2h0194:79744:80434 [0] NCCL INFO Setting affinity for GPU 3 to ff,ffff0000,00ffffff\n", - "Mon Sep 6 09:04:57 2021[1,2]:dgx2h0194:79722:80271 [0] NCCL INFO Trees [0] 7/-1/-1->2->5 [1] 7/-1/-1->2->5 [2] 7/-1/-1->2->5 [3] 7/-1/-1->2->5 [4] 7/-1/-1->2->5 [5] 7/-1/-1->2->5 [6] 7/-1/-1->2->5 [7] 7/-1/-1->2->5 [8] 7/-1/-1->2->5 [9] 7/-1/-1->2->5 [10] 7/-1/-1->2->5 [11] 7/-1/-1->2->5\n", - "Mon Sep 6 09:04:57 2021[1,2]:dgx2h0194:79722:80271 [0] NCCL INFO Setting affinity for GPU 8 to ffffff00,0000ffff,ff000000\n", - "Mon Sep 6 09:04:57 2021[1,5]:dgx2h0194:79738:80437 [0] NCCL INFO Trees [0] 2/-1/-1->5->1 [1] 2/-1/-1->5->1 [2] 2/-1/-1->5->1 [3] 2/-1/-1->5->1 [4] 2/-1/-1->5->1 [5] 2/-1/-1->5->1 [6] 2/-1/-1->5->1 [7] 2/-1/-1->5->1 [8] 2/-1/-1->5->1 [9] 2/-1/-1->5->1 [10] 2/-1/-1->5->1 [11] 2/-1/-1->5->1\n", - "Mon Sep 6 09:04:57 2021[1,5]:dgx2h0194:79738:80437 [0] NCCL INFO Setting affinity for GPU 10 to ffffff00,0000ffff,ff000000\n", - "Mon Sep 6 09:04:57 2021[1,7]:dgx2h0194:79740:80605 [0] NCCL INFO Trees [0] 3/-1/-1->7->2 [1] 3/-1/-1->7->2 [2] 3/-1/-1->7->2 [3] 3/-1/-1->7->2 [4] 3/-1/-1->7->2 [5] 3/-1/-1->7->2 [6] 3/-1/-1->7->2 [7] 3/-1/-1->7->2 [8] 3/-1/-1->7->2 [9] 3/-1/-1->7->2 [10] 3/-1/-1->7->2 [11] 3/-1/-1->7->2\n", - "Mon Sep 6 09:04:57 2021[1,7]:dgx2h0194:79740:80605 [0] NCCL INFO Setting affinity for GPU 9 to ffffff00,0000ffff,ff000000\n", - "Mon Sep 6 09:04:57 2021[1,3]:dgx2h0194:79727:80587 [0] NCCL INFO Trees [0] 15/-1/-1->3->7 [1] 15/-1/-1->3->7 [2] 15/-1/-1->3->7 [3] 15/-1/-1->3->7 [4] 15/-1/-1->3->7 [5] 15/-1/-1->3->7 [6] 15/-1/-1->3->7 [7] 15/-1/-1->3->7 [8] 15/-1/-1->3->7 [9] 15/-1/-1->3->7 [10] 15/-1/-1->3->7 [11] 15/-1/-1->3->7\n", - "Mon Sep 6 09:04:57 2021[1,3]:dgx2h0194:79727:80587 [0] NCCL INFO Setting affinity for GPU 15 to ffffff00,0000ffff,ff000000\n", - "Mon Sep 6 09:04:57 2021[1,0]:dgx2h0194:79712:80584 [0] NCCL INFO Setting affinity for GPU 1 to ff,ffff0000,00ffffff\n", - "Mon Sep 6 09:04:57 2021[1,4]:dgx2h0194:79732:80427 [0] NCCL INFO Trees [0] 14/-1/-1->4->15 [1] 14/-1/-1->4->15 [2] 14/-1/-1->4->15 [3] 14/-1/-1->4->15 [4] 14/-1/-1->4->15 [5] 14/-1/-1->4->15 [6] 14/-1/-1->4->15 [7] 14/-1/-1->4->15 [8] 14/-1/-1->4->15 [9] 14/-1/-1->4->15 [10] 14/-1/-1->4->15 [11] 14/-1/-1->4->15\n", - "Mon Sep 6 09:04:57 2021[1,4]:dgx2h0194:79732:80427 [0] NCCL INFO Setting affinity for GPU 13 to ffffff00,0000ffff,ff000000\n", - "Mon Sep 6 09:04:57 2021[1,6]:dgx2h0194:79739:80608 [0] NCCL INFO Trees [0] 10/-1/-1->6->12 [1] 10/-1/-1->6->12 [2] 10/-1/-1->6->12 [3] 10/-1/-1->6->12 [4] 10/-1/-1->6->12 [5] 10/-1/-1->6->12 [6] 10/-1/-1->6->12 [7] 10/-1/-1->6->12 [8] 10/-1/-1->6->12 [9] 10/-1/-1->6->12 [10] 10/-1/-1->6->12 [11] 10/-1/-1->6->12\n", - "Mon Sep 6 09:04:57 2021[1,6]:dgx2h0194:79739:80608 [0] NCCL INFO Setting affinity for GPU 6 to ff,ffff0000,00ffffff\n", - "Mon Sep 6 09:04:57 2021[1,8]:dgx2h0194:79741:80747 [0] NCCL INFO Trees [0] 11/-1/-1->8->0 [1] 11/-1/-1->8->0 [2] 11/-1/-1->8->0 [3] 11/-1/-1->8->0 [4] 11/-1/-1->8->0 [5] 11/-1/-1->8->0 [6] 11/-1/-1->8->0 [7] 11/-1/-1->8->0 [8] 11/-1/-1->8->0 [9] 11/-1/-1->8->0 [10] 11/-1/-1->8->0 [11] 11/-1/-1->8->0\n", - "Mon Sep 6 09:04:57 2021[1,8]:dgx2h0194:79741:80747 [0] NCCL INFO Setting affinity for GPU 0 to ff,ffff0000,00ffffff\n", - "Mon Sep 6 09:04:57 2021[1,9]:dgx2h0194:79742:80744 [0] NCCL INFO Trees [0] 13/-1/-1->9->10 [1] 13/-1/-1->9->10 [2] 13/-1/-1->9->10 [3] 13/-1/-1->9->10 [4] 13/-1/-1->9->10 [5] 13/-1/-1->9->10 [6] 13/-1/-1->9->10 [7] 13/-1/-1->9->10 [8] 13/-1/-1->9->10 [9] 13/-1/-1->9->10 [10] 13/-1/-1->9->10 [11] 13/-1/-1->9->10\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:57 2021[1,9]:dgx2h0194:79742:80744 [0] NCCL INFO Setting affinity for GPU 5 to ff,ffff0000,00ffffff\n", - "Mon Sep 6 09:04:57 2021[1,10]:dgx2h0194:79743:80602 [0] NCCL INFO Trees [0] 9/-1/-1->10->6 [1] 9/-1/-1->10->6 [2] 9/-1/-1->10->6 [3] 9/-1/-1->10->6 [4] 9/-1/-1->10->6 [5] 9/-1/-1->10->6 [6] 9/-1/-1->10->6 [7] 9/-1/-1->10->6 [8] 9/-1/-1->10->6 [9] 9/-1/-1->10->6 [10] 9/-1/-1->10->6 [11] 9/-1/-1->10->6\n", - "Mon Sep 6 09:04:57 2021[1,10]:dgx2h0194:79743:80602 [0] NCCL INFO Setting affinity for GPU 7 to ff,ffff0000,00ffffff\n", - "Mon Sep 6 09:04:57 2021[1,15]:dgx2h0194:79748:80424 [0] NCCL INFO Channel 00 : 15[e5000] -> 4[e2000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,6]:dgx2h0194:79739:80608 [0] NCCL INFO Channel 00 : 6[5c000] -> 10[5e000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,8]:dgx2h0194:79741:80747 [0] NCCL INFO Channel 00 : 8[34000] -> 11[3b000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,9]:dgx2h0194:79742:80744 [0] NCCL INFO Channel 00 : 9[59000] -> 13[57000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,11]:dgx2h0194:79744:80434 [0] NCCL INFO Channel 00 : 11[3b000] -> 12[39000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,14]:dgx2h0194:79747:80615 [0] NCCL INFO Channel 00 : 14[e0000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,2]:dgx2h0194:79722:80271 [0] NCCL INFO Channel 00 : 2[b7000] -> 7[b9000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,15]:dgx2h0194:79748:80424 [0] NCCL INFO Channel 01 : 15[e5000] -> 4[e2000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,6]:dgx2h0194:79739:80608 [0] NCCL INFO Channel 01 : 6[5c000] -> 10[5e000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,9]:dgx2h0194:79742:80744 [0] NCCL INFO Channel 01 : 9[59000] -> 13[57000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,8]:dgx2h0194:79741:80747 [0] NCCL INFO Channel 01 : 8[34000] -> 11[3b000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,11]:dgx2h0194:79744:80434 [0] NCCL INFO Channel 01 : 11[3b000] -> 12[39000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,14]:dgx2h0194:79747:80615 [0] NCCL INFO Channel 01 : 14[e0000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,2]:dgx2h0194:79722:80271 [0] NCCL INFO Channel 01 : 2[b7000] -> 7[b9000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,15]:dgx2h0194:79748:80424 [0] NCCL INFO Channel 02 : 15[e5000] -> 4[e2000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,6]:dgx2h0194:79739:80608 [0] NCCL INFO Channel 02 : 6[5c000] -> 10[5e000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,8]:dgx2h0194:79741:80747 [0] NCCL INFO Channel 02 : 8[34000] -> 11[3b000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,9]:dgx2h0194:79742:80744 [0] NCCL INFO Channel 02 : 9[59000] -> 13[57000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,11]:dgx2h0194:79744:80434 [0] NCCL INFO Channel 02 : 11[3b000] -> 12[39000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,14]:dgx2h0194:79747:80615 [0] NCCL INFO Channel 02 : 14[e0000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,2]:dgx2h0194:79722:80271 [0] NCCL INFO Channel 02 : 2[b7000] -> 7[b9000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,15]:dgx2h0194:79748:80424 [0] NCCL INFO Channel 03 : 15[e5000] -> 4[e2000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,6]:dgx2h0194:79739:80608 [0] NCCL INFO Channel 03 : 6[5c000] -> 10[5e000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,9]:dgx2h0194:79742:80744 [0] NCCL INFO Channel 03 : 9[59000] -> 13[57000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,8]:dgx2h0194:79741:80747 [0] NCCL INFO Channel 03 : 8[34000] -> 11[3b000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,11]:dgx2h0194:79744:80434 [0] NCCL INFO Channel 03 : 11[3b000] -> 12[39000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,14]:dgx2h0194:79747:80615 [0] NCCL INFO Channel 03 : 14[e0000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,2]:dgx2h0194:79722:80271 [0] NCCL INFO Channel 03 : 2[b7000] -> 7[b9000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,15]:dgx2h0194:79748:80424 [0] NCCL INFO Channel 04 : 15[e5000] -> 4[e2000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,6]:dgx2h0194:79739:80608 [0] NCCL INFO Channel 04 : 6[5c000] -> 10[5e000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,8]:dgx2h0194:79741:80747 [0] NCCL INFO Channel 04 : 8[34000] -> 11[3b000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,9]:dgx2h0194:79742:80744 [0] NCCL INFO Channel 04 : 9[59000] -> 13[57000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,11]:dgx2h0194:79744:80434 [0] NCCL INFO Channel 04 : 11[3b000] -> 12[39000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,14]:dgx2h0194:79747:80615 [0] NCCL INFO Channel 04 : 14[e0000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,2]:dgx2h0194:79722:80271 [0] NCCL INFO Channel 04 : 2[b7000] -> 7[b9000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,15]:dgx2h0194:79748:80424 [0] NCCL INFO Channel 05 : 15[e5000] -> 4[e2000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,6]:dgx2h0194:79739:80608 [0] NCCL INFO Channel 05 : 6[5c000] -> 10[5e000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,9]:dgx2h0194:79742:80744 [0] NCCL INFO Channel 05 : 9[59000] -> 13[57000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,8]:dgx2h0194:79741:80747 [0] NCCL INFO Channel 05 : 8[34000] -> 11[3b000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,11]:dgx2h0194:79744:80434 [0] NCCL INFO Channel 05 : 11[3b000] -> 12[39000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,14]:dgx2h0194:79747:80615 [0] NCCL INFO Channel 05 : 14[e0000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,2]:dgx2h0194:79722:80271 [0] NCCL INFO Channel 05 : 2[b7000] -> 7[b9000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,15]:dgx2h0194:79748:80424 [0] NCCL INFO Channel 06 : 15[e5000] -> 4[e2000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,6]:dgx2h0194:79739:80608 [0] NCCL INFO Channel 06 : 6[5c000] -> 10[5e000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,9]:dgx2h0194:79742:80744 [0] NCCL INFO Channel 06 : 9[59000] -> 13[57000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,8]:dgx2h0194:79741:80747 [0] NCCL INFO Channel 06 : 8[34000] -> 11[3b000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,11]:dgx2h0194:79744:80434 [0] NCCL INFO Channel 06 : 11[3b000] -> 12[39000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,14]:dgx2h0194:79747:80615 [0] NCCL INFO Channel 06 : 14[e0000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,2]:dgx2h0194:79722:80271 [0] NCCL INFO Channel 06 : 2[b7000] -> 7[b9000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,15]:dgx2h0194:79748:80424 [0] NCCL INFO Channel 07 : 15[e5000] -> 4[e2000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,6]:dgx2h0194:79739:80608 [0] NCCL INFO Channel 07 : 6[5c000] -> 10[5e000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,9]:dgx2h0194:79742:80744 [0] NCCL INFO Channel 07 : 9[59000] -> 13[57000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,8]:dgx2h0194:79741:80747 [0] NCCL INFO Channel 07 : 8[34000] -> 11[3b000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,11]:dgx2h0194:79744:80434 [0] NCCL INFO Channel 07 : 11[3b000] -> 12[39000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,14]:dgx2h0194:79747:80615 [0] NCCL INFO Channel 07 : 14[e0000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,2]:dgx2h0194:79722:80271 [0] NCCL INFO Channel 07 : 2[b7000] -> 7[b9000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,15]:dgx2h0194:79748:80424 [0] NCCL INFO Channel 08 : 15[e5000] -> 4[e2000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,6]:dgx2h0194:79739:80608 [0] NCCL INFO Channel 08 : 6[5c000] -> 10[5e000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,8]:dgx2h0194:79741:80747 [0] NCCL INFO Channel 08 : 8[34000] -> 11[3b000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,9]:dgx2h0194:79742:80744 [0] NCCL INFO Channel 08 : 9[59000] -> 13[57000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,11]:dgx2h0194:79744:80434 [0] NCCL INFO Channel 08 : 11[3b000] -> 12[39000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,14]:dgx2h0194:79747:80615 [0] NCCL INFO Channel 08 : 14[e0000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,2]:dgx2h0194:79722:80271 [0] NCCL INFO Channel 08 : 2[b7000] -> 7[b9000] via P2P/IPC\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:57 2021[1,15]:dgx2h0194:79748:80424 [0] NCCL INFO Channel 09 : 15[e5000] -> 4[e2000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,6]:dgx2h0194:79739:80608 [0] NCCL INFO Channel 09 : 6[5c000] -> 10[5e000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,9]:dgx2h0194:79742:80744 [0] NCCL INFO Channel 09 : 9[59000] -> 13[57000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,8]:dgx2h0194:79741:80747 [0] NCCL INFO Channel 09 : 8[34000] -> 11[3b000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,11]:dgx2h0194:79744:80434 [0] NCCL INFO Channel 09 : 11[3b000] -> 12[39000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,14]:dgx2h0194:79747:80615 [0] NCCL INFO Channel 09 : 14[e0000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,2]:dgx2h0194:79722:80271 [0] NCCL INFO Channel 09 : 2[b7000] -> 7[b9000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,15]:dgx2h0194:79748:80424 [0] NCCL INFO Channel 10 : 15[e5000] -> 4[e2000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,6]:dgx2h0194:79739:80608 [0] NCCL INFO Channel 10 : 6[5c000] -> 10[5e000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,9]:dgx2h0194:79742:80744 [0] NCCL INFO Channel 10 : 9[59000] -> 13[57000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,8]:dgx2h0194:79741:80747 [0] NCCL INFO Channel 10 : 8[34000] -> 11[3b000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,11]:dgx2h0194:79744:80434 [0] NCCL INFO Channel 10 : 11[3b000] -> 12[39000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,14]:dgx2h0194:79747:80615 [0] NCCL INFO Channel 10 : 14[e0000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,2]:dgx2h0194:79722:80271 [0] NCCL INFO Channel 10 : 2[b7000] -> 7[b9000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,15]:dgx2h0194:79748:80424 [0] NCCL INFO Channel 11 : 15[e5000] -> 4[e2000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,6]:dgx2h0194:79739:80608 [0] NCCL INFO Channel 11 : 6[5c000] -> 10[5e000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,9]:dgx2h0194:79742:80744 [0] NCCL INFO Channel 11 : 9[59000] -> 13[57000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,8]:dgx2h0194:79741:80747 [0] NCCL INFO Channel 11 : 8[34000] -> 11[3b000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,11]:dgx2h0194:79744:80434 [0] NCCL INFO Channel 11 : 11[3b000] -> 12[39000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,14]:dgx2h0194:79747:80615 [0] NCCL INFO Channel 11 : 14[e0000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,2]:dgx2h0194:79722:80271 [0] NCCL INFO Channel 11 : 2[b7000] -> 7[b9000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,13]:dgx2h0194:79746:80607 [0] NCCL INFO Channel 00 : 13[57000] -> 1[be000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,3]:dgx2h0194:79727:80587 [0] NCCL INFO Channel 00 : 3[e7000] -> 15[e5000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,1]:dgx2h0194:79717:80738 [0] NCCL INFO Channel 00 : 1[be000] -> 5[bc000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,13]:dgx2h0194:79746:80607 [0] NCCL INFO Channel 01 : 13[57000] -> 1[be000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,3]:dgx2h0194:79727:80587 [0] NCCL INFO Channel 01 : 3[e7000] -> 15[e5000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,1]:dgx2h0194:79717:80738 [0] NCCL INFO Channel 01 : 1[be000] -> 5[bc000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,13]:dgx2h0194:79746:80607 [0] NCCL INFO Channel 02 : 13[57000] -> 1[be000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,3]:dgx2h0194:79727:80587 [0] NCCL INFO Channel 02 : 3[e7000] -> 15[e5000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,1]:dgx2h0194:79717:80738 [0] NCCL INFO Channel 02 : 1[be000] -> 5[bc000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,13]:dgx2h0194:79746:80607 [0] NCCL INFO Channel 03 : 13[57000] -> 1[be000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,3]:dgx2h0194:79727:80587 [0] NCCL INFO Channel 03 : 3[e7000] -> 15[e5000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,1]:dgx2h0194:79717:80738 [0] NCCL INFO Channel 03 : 1[be000] -> 5[bc000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,13]:dgx2h0194:79746:80607 [0] NCCL INFO Channel 04 : 13[57000] -> 1[be000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,3]:dgx2h0194:79727:80587 [0] NCCL INFO Channel 04 : 3[e7000] -> 15[e5000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,13]:dgx2h0194:79746:80607 [0] NCCL INFO Channel 05 : 13[57000] -> 1[be000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,1]:dgx2h0194:79717:80738 [0] NCCL INFO Channel 04 : 1[be000] -> 5[bc000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,3]:dgx2h0194:79727:80587 [0] NCCL INFO Channel 05 : 3[e7000] -> 15[e5000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,1]:dgx2h0194:79717:80738 [0] NCCL INFO Channel 05 : 1[be000] -> 5[bc000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,13]:dgx2h0194:79746:80607 [0] NCCL INFO Channel 06 : 13[57000] -> 1[be000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,3]:dgx2h0194:79727:80587 [0] NCCL INFO Channel 06 : 3[e7000] -> 15[e5000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,13]:dgx2h0194:79746:80607 [0] NCCL INFO Channel 07 : 13[57000] -> 1[be000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,1]:dgx2h0194:79717:80738 [0] NCCL INFO Channel 06 : 1[be000] -> 5[bc000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,3]:dgx2h0194:79727:80587 [0] NCCL INFO Channel 07 : 3[e7000] -> 15[e5000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,1]:dgx2h0194:79717:80738 [0] NCCL INFO Channel 07 : 1[be000] -> 5[bc000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,13]:dgx2h0194:79746:80607 [0] NCCL INFO Channel 08 : 13[57000] -> 1[be000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,3]:dgx2h0194:79727:80587 [0] NCCL INFO Channel 08 : 3[e7000] -> 15[e5000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,1]:dgx2h0194:79717:80738 [0] NCCL INFO Channel 08 : 1[be000] -> 5[bc000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,13]:dgx2h0194:79746:80607 [0] NCCL INFO Channel 09 : 13[57000] -> 1[be000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,1]:dgx2h0194:79717:80738 [0] NCCL INFO Channel 09 : 1[be000] -> 5[bc000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,3]:dgx2h0194:79727:80587 [0] NCCL INFO Channel 09 : 3[e7000] -> 15[e5000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,13]:dgx2h0194:79746:80607 [0] NCCL INFO Channel 10 : 13[57000] -> 1[be000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,3]:dgx2h0194:79727:80587 [0] NCCL INFO Channel 10 : 3[e7000] -> 15[e5000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,1]:dgx2h0194:79717:80738 [0] NCCL INFO Channel 10 : 1[be000] -> 5[bc000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,13]:dgx2h0194:79746:80607 [0] NCCL INFO Channel 11 : 13[57000] -> 1[be000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,1]:dgx2h0194:79717:80738 [0] NCCL INFO Channel 11 : 1[be000] -> 5[bc000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,3]:dgx2h0194:79727:80587 [0] NCCL INFO Channel 11 : 3[e7000] -> 15[e5000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,12]:dgx2h0194:79745:80741 [0] NCCL INFO Channel 00 : 12[39000] -> 6[5c000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,12]:dgx2h0194:79745:80741 [0] NCCL INFO Channel 01 : 12[39000] -> 6[5c000] via P2P/IPC\n", - "Mon Sep 6 09:04:57 2021[1,12]:dgx2h0194:79745:80741 [0] NCCL INFO Channel 02 : 12[39000] -> 6[5c000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,12]:dgx2h0194:79745:80741 [0] NCCL INFO Channel 03 : 12[39000] -> 6[5c000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,10]:dgx2h0194:79743:80602 [0] NCCL INFO Channel 00 : 10[5e000] -> 9[59000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,12]:dgx2h0194:79745:80741 [0] NCCL INFO Channel 04 : 12[39000] -> 6[5c000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,10]:dgx2h0194:79743:80602 [0] NCCL INFO Channel 01 : 10[5e000] -> 9[59000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,12]:dgx2h0194:79745:80741 [0] NCCL INFO Channel 05 : 12[39000] -> 6[5c000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,10]:dgx2h0194:79743:80602 [0] NCCL INFO Channel 02 : 10[5e000] -> 9[59000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,12]:dgx2h0194:79745:80741 [0] NCCL INFO Channel 06 : 12[39000] -> 6[5c000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,10]:dgx2h0194:79743:80602 [0] NCCL INFO Channel 03 : 10[5e000] -> 9[59000] via P2P/IPC\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:58 2021[1,12]:dgx2h0194:79745:80741 [0] NCCL INFO Channel 07 : 12[39000] -> 6[5c000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,10]:dgx2h0194:79743:80602 [0] NCCL INFO Channel 04 : 10[5e000] -> 9[59000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,12]:dgx2h0194:79745:80741 [0] NCCL INFO Channel 08 : 12[39000] -> 6[5c000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,10]:dgx2h0194:79743:80602 [0] NCCL INFO Channel 05 : 10[5e000] -> 9[59000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,12]:dgx2h0194:79745:80741 [0] NCCL INFO Channel 09 : 12[39000] -> 6[5c000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,10]:dgx2h0194:79743:80602 [0] NCCL INFO Channel 06 : 10[5e000] -> 9[59000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,7]:dgx2h0194:79740:80605 [0] NCCL INFO Channel 00 : 7[b9000] -> 3[e7000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,12]:dgx2h0194:79745:80741 [0] NCCL INFO Channel 10 : 12[39000] -> 6[5c000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,10]:dgx2h0194:79743:80602 [0] NCCL INFO Channel 07 : 10[5e000] -> 9[59000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,7]:dgx2h0194:79740:80605 [0] NCCL INFO Channel 01 : 7[b9000] -> 3[e7000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,12]:dgx2h0194:79745:80741 [0] NCCL INFO Channel 11 : 12[39000] -> 6[5c000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,10]:dgx2h0194:79743:80602 [0] NCCL INFO Channel 08 : 10[5e000] -> 9[59000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,7]:dgx2h0194:79740:80605 [0] NCCL INFO Channel 02 : 7[b9000] -> 3[e7000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,10]:dgx2h0194:79743:80602 [0] NCCL INFO Channel 09 : 10[5e000] -> 9[59000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,7]:dgx2h0194:79740:80605 [0] NCCL INFO Channel 03 : 7[b9000] -> 3[e7000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,10]:dgx2h0194:79743:80602 [0] NCCL INFO Channel 10 : 10[5e000] -> 9[59000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,7]:dgx2h0194:79740:80605 [0] NCCL INFO Channel 04 : 7[b9000] -> 3[e7000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,10]:dgx2h0194:79743:80602 [0] NCCL INFO Channel 11 : 10[5e000] -> 9[59000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,7]:dgx2h0194:79740:80605 [0] NCCL INFO Channel 05 : 7[b9000] -> 3[e7000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,7]:dgx2h0194:79740:80605 [0] NCCL INFO Channel 06 : 7[b9000] -> 3[e7000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,0]:dgx2h0194:79712:80584 [0] NCCL INFO Channel 00 : 0[36000] -> 8[34000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,7]:dgx2h0194:79740:80605 [0] NCCL INFO Channel 07 : 7[b9000] -> 3[e7000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,0]:dgx2h0194:79712:80584 [0] NCCL INFO Channel 01 : 0[36000] -> 8[34000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,7]:dgx2h0194:79740:80605 [0] NCCL INFO Channel 08 : 7[b9000] -> 3[e7000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,0]:dgx2h0194:79712:80584 [0] NCCL INFO Channel 02 : 0[36000] -> 8[34000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,7]:dgx2h0194:79740:80605 [0] NCCL INFO Channel 09 : 7[b9000] -> 3[e7000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,0]:dgx2h0194:79712:80584 [0] NCCL INFO Channel 03 : 0[36000] -> 8[34000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,7]:dgx2h0194:79740:80605 [0] NCCL INFO Channel 10 : 7[b9000] -> 3[e7000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,0]:dgx2h0194:79712:80584 [0] NCCL INFO Channel 04 : 0[36000] -> 8[34000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,7]:dgx2h0194:79740:80605 [0] NCCL INFO Channel 11 : 7[b9000] -> 3[e7000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,0]:dgx2h0194:79712:80584 [0] NCCL INFO Channel 05 : 0[36000] -> 8[34000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,0]:dgx2h0194:79712:80584 [0] NCCL INFO Channel 06 : 0[36000] -> 8[34000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,4]:dgx2h0194:79732:80427 [0] NCCL INFO Channel 00 : 4[e2000] -> 14[e0000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,0]:dgx2h0194:79712:80584 [0] NCCL INFO Channel 07 : 0[36000] -> 8[34000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,4]:dgx2h0194:79732:80427 [0] NCCL INFO Channel 01 : 4[e2000] -> 14[e0000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,0]:dgx2h0194:79712:80584 [0] NCCL INFO Channel 08 : 0[36000] -> 8[34000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,4]:dgx2h0194:79732:80427 [0] NCCL INFO Channel 02 : 4[e2000] -> 14[e0000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,0]:dgx2h0194:79712:80584 [0] NCCL INFO Channel 09 : 0[36000] -> 8[34000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,4]:dgx2h0194:79732:80427 [0] NCCL INFO Channel 03 : 4[e2000] -> 14[e0000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,0]:dgx2h0194:79712:80584 [0] NCCL INFO Channel 10 : 0[36000] -> 8[34000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,4]:dgx2h0194:79732:80427 [0] NCCL INFO Channel 04 : 4[e2000] -> 14[e0000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,0]:dgx2h0194:79712:80584 [0] NCCL INFO Channel 11 : 0[36000] -> 8[34000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,4]:dgx2h0194:79732:80427 [0] NCCL INFO Channel 05 : 4[e2000] -> 14[e0000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,4]:dgx2h0194:79732:80427 [0] NCCL INFO Channel 06 : 4[e2000] -> 14[e0000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,4]:dgx2h0194:79732:80427 [0] NCCL INFO Channel 07 : 4[e2000] -> 14[e0000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,4]:dgx2h0194:79732:80427 [0] NCCL INFO Channel 08 : 4[e2000] -> 14[e0000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,4]:dgx2h0194:79732:80427 [0] NCCL INFO Channel 09 : 4[e2000] -> 14[e0000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,4]:dgx2h0194:79732:80427 [0] NCCL INFO Channel 10 : 4[e2000] -> 14[e0000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,4]:dgx2h0194:79732:80427 [0] NCCL INFO Channel 11 : 4[e2000] -> 14[e0000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,5]:dgx2h0194:79738:80437 [0] NCCL INFO Channel 00 : 5[bc000] -> 2[b7000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,5]:dgx2h0194:79738:80437 [0] NCCL INFO Channel 01 : 5[bc000] -> 2[b7000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,5]:dgx2h0194:79738:80437 [0] NCCL INFO Channel 02 : 5[bc000] -> 2[b7000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,11]:dgx2h0194:79744:80434 [0] NCCL INFO Connected all rings\n", - "Mon Sep 6 09:04:58 2021[1,5]:dgx2h0194:79738:80437 [0] NCCL INFO Channel 03 : 5[bc000] -> 2[b7000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,11]:dgx2h0194:79744:80434 [0] NCCL INFO Channel 00 : 11[3b000] -> 8[34000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,5]:dgx2h0194:79738:80437 [0] NCCL INFO Channel 04 : 5[bc000] -> 2[b7000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,11]:dgx2h0194:79744:80434 [0] NCCL INFO Channel 01 : 11[3b000] -> 8[34000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,5]:dgx2h0194:79738:80437 [0] NCCL INFO Channel 05 : 5[bc000] -> 2[b7000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,11]:dgx2h0194:79744:80434 [0] NCCL INFO Channel 02 : 11[3b000] -> 8[34000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,5]:dgx2h0194:79738:80437 [0] NCCL INFO Channel 06 : 5[bc000] -> 2[b7000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,11]:dgx2h0194:79744:80434 [0] NCCL INFO Channel 03 : 11[3b000] -> 8[34000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,5]:dgx2h0194:79738:80437 [0] NCCL INFO Channel 07 : 5[bc000] -> 2[b7000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,11]:dgx2h0194:79744:80434 [0] NCCL INFO Channel 04 : 11[3b000] -> 8[34000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,5]:dgx2h0194:79738:80437 [0] NCCL INFO Channel 08 : 5[bc000] -> 2[b7000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,11]:dgx2h0194:79744:80434 [0] NCCL INFO Channel 05 : 11[3b000] -> 8[34000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,6]:dgx2h0194:79739:80608 [0] NCCL INFO Connected all rings\n", - "Mon Sep 6 09:04:58 2021[1,5]:dgx2h0194:79738:80437 [0] NCCL INFO Channel 09 : 5[bc000] -> 2[b7000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,11]:dgx2h0194:79744:80434 [0] NCCL INFO Channel 06 : 11[3b000] -> 8[34000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,6]:dgx2h0194:79739:80608 [0] NCCL INFO Channel 00 : 6[5c000] -> 12[39000] via P2P/IPC\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:58 2021[1,5]:dgx2h0194:79738:80437 [0] NCCL INFO Channel 10 : 5[bc000] -> 2[b7000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,11]:dgx2h0194:79744:80434 [0] NCCL INFO Channel 07 : 11[3b000] -> 8[34000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,6]:dgx2h0194:79739:80608 [0] NCCL INFO Channel 01 : 6[5c000] -> 12[39000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,5]:dgx2h0194:79738:80437 [0] NCCL INFO Channel 11 : 5[bc000] -> 2[b7000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,11]:dgx2h0194:79744:80434 [0] NCCL INFO Channel 08 : 11[3b000] -> 8[34000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,6]:dgx2h0194:79739:80608 [0] NCCL INFO Channel 02 : 6[5c000] -> 12[39000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,12]:dgx2h0194:79745:80741 [0] NCCL INFO Connected all rings\n", - "Mon Sep 6 09:04:58 2021[1,11]:dgx2h0194:79744:80434 [0] NCCL INFO Channel 09 : 11[3b000] -> 8[34000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,6]:dgx2h0194:79739:80608 [0] NCCL INFO Channel 03 : 6[5c000] -> 12[39000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,11]:dgx2h0194:79744:80434 [0] NCCL INFO Channel 10 : 11[3b000] -> 8[34000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,6]:dgx2h0194:79739:80608 [0] NCCL INFO Channel 04 : 6[5c000] -> 12[39000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,11]:dgx2h0194:79744:80434 [0] NCCL INFO Channel 11 : 11[3b000] -> 8[34000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,6]:dgx2h0194:79739:80608 [0] NCCL INFO Channel 05 : 6[5c000] -> 12[39000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,7]:dgx2h0194:79740:80605 [0] NCCL INFO Connected all rings\n", - "Mon Sep 6 09:04:58 2021[1,6]:dgx2h0194:79739:80608 [0] NCCL INFO Channel 06 : 6[5c000] -> 12[39000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,6]:dgx2h0194:79739:80608 [0] NCCL INFO Channel 07 : 6[5c000] -> 12[39000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,6]:dgx2h0194:79739:80608 [0] NCCL INFO Channel 08 : 6[5c000] -> 12[39000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,1]:dgx2h0194:79717:80738 [0] NCCL INFO Connected all rings\n", - "Mon Sep 6 09:04:58 2021[1,6]:dgx2h0194:79739:80608 [0] NCCL INFO Channel 09 : 6[5c000] -> 12[39000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,6]:dgx2h0194:79739:80608 [0] NCCL INFO Channel 10 : 6[5c000] -> 12[39000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,6]:dgx2h0194:79739:80608 [0] NCCL INFO Channel 11 : 6[5c000] -> 12[39000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,13]:dgx2h0194:79746:80607 [0] NCCL INFO Connected all rings\n", - "Mon Sep 6 09:04:58 2021[1,5]:dgx2h0194:79738:80437 [0] NCCL INFO Connected all rings\n", - "Mon Sep 6 09:04:58 2021[1,15]:dgx2h0194:79748:80424 [0] NCCL INFO Connected all rings\n", - "Mon Sep 6 09:04:58 2021[1,1]:dgx2h0194:79717:80738 [0] NCCL INFO Channel 00 : 1[be000] -> 13[57000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,15]:dgx2h0194:79748:80424 [0] NCCL INFO Channel 00 : 15[e5000] -> 3[e7000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,12]:dgx2h0194:79745:80741 [0] NCCL INFO Channel 00 : 12[39000] -> 11[3b000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,1]:dgx2h0194:79717:80738 [0] NCCL INFO Channel 01 : 1[be000] -> 13[57000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,15]:dgx2h0194:79748:80424 [0] NCCL INFO Channel 01 : 15[e5000] -> 3[e7000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,12]:dgx2h0194:79745:80741 [0] NCCL INFO Channel 01 : 12[39000] -> 11[3b000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,1]:dgx2h0194:79717:80738 [0] NCCL INFO Channel 02 : 1[be000] -> 13[57000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,15]:dgx2h0194:79748:80424 [0] NCCL INFO Channel 02 : 15[e5000] -> 3[e7000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,4]:dgx2h0194:79732:80427 [0] NCCL INFO Connected all rings\n", - "Mon Sep 6 09:04:58 2021[1,12]:dgx2h0194:79745:80741 [0] NCCL INFO Channel 02 : 12[39000] -> 11[3b000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,13]:dgx2h0194:79746:80607 [0] NCCL INFO Channel 00 : 13[57000] -> 9[59000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,1]:dgx2h0194:79717:80738 [0] NCCL INFO Channel 03 : 1[be000] -> 13[57000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,15]:dgx2h0194:79748:80424 [0] NCCL INFO Channel 03 : 15[e5000] -> 3[e7000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,12]:dgx2h0194:79745:80741 [0] NCCL INFO Channel 03 : 12[39000] -> 11[3b000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,13]:dgx2h0194:79746:80607 [0] NCCL INFO Channel 01 : 13[57000] -> 9[59000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,1]:dgx2h0194:79717:80738 [0] NCCL INFO Channel 04 : 1[be000] -> 13[57000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,15]:dgx2h0194:79748:80424 [0] NCCL INFO Channel 04 : 15[e5000] -> 3[e7000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,12]:dgx2h0194:79745:80741 [0] NCCL INFO Channel 04 : 12[39000] -> 11[3b000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,13]:dgx2h0194:79746:80607 [0] NCCL INFO Channel 02 : 13[57000] -> 9[59000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,1]:dgx2h0194:79717:80738 [0] NCCL INFO Channel 05 : 1[be000] -> 13[57000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,15]:dgx2h0194:79748:80424 [0] NCCL INFO Channel 05 : 15[e5000] -> 3[e7000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,10]:dgx2h0194:79743:80602 [0] NCCL INFO Connected all rings\n", - "Mon Sep 6 09:04:58 2021[1,12]:dgx2h0194:79745:80741 [0] NCCL INFO Channel 05 : 12[39000] -> 11[3b000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,1]:dgx2h0194:79717:80738 [0] NCCL INFO Channel 06 : 1[be000] -> 13[57000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,13]:dgx2h0194:79746:80607 [0] NCCL INFO Channel 03 : 13[57000] -> 9[59000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,15]:dgx2h0194:79748:80424 [0] NCCL INFO Channel 06 : 15[e5000] -> 3[e7000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,12]:dgx2h0194:79745:80741 [0] NCCL INFO Channel 06 : 12[39000] -> 11[3b000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,13]:dgx2h0194:79746:80607 [0] NCCL INFO Channel 04 : 13[57000] -> 9[59000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,1]:dgx2h0194:79717:80738 [0] NCCL INFO Channel 07 : 1[be000] -> 13[57000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,15]:dgx2h0194:79748:80424 [0] NCCL INFO Channel 07 : 15[e5000] -> 3[e7000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,12]:dgx2h0194:79745:80741 [0] NCCL INFO Channel 07 : 12[39000] -> 11[3b000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,13]:dgx2h0194:79746:80607 [0] NCCL INFO Channel 05 : 13[57000] -> 9[59000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,1]:dgx2h0194:79717:80738 [0] NCCL INFO Channel 08 : 1[be000] -> 13[57000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,15]:dgx2h0194:79748:80424 [0] NCCL INFO Channel 08 : 15[e5000] -> 3[e7000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,14]:dgx2h0194:79747:80615 [0] NCCL INFO Connected all rings\n", - "Mon Sep 6 09:04:58 2021[1,12]:dgx2h0194:79745:80741 [0] NCCL INFO Channel 08 : 12[39000] -> 11[3b000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,13]:dgx2h0194:79746:80607 [0] NCCL INFO Channel 06 : 13[57000] -> 9[59000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,1]:dgx2h0194:79717:80738 [0] NCCL INFO Channel 09 : 1[be000] -> 13[57000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,15]:dgx2h0194:79748:80424 [0] NCCL INFO Channel 09 : 15[e5000] -> 3[e7000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,14]:dgx2h0194:79747:80615 [0] NCCL INFO Channel 00 : 14[e0000] -> 4[e2000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,12]:dgx2h0194:79745:80741 [0] NCCL INFO Channel 09 : 12[39000] -> 11[3b000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,13]:dgx2h0194:79746:80607 [0] NCCL INFO Channel 07 : 13[57000] -> 9[59000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,1]:dgx2h0194:79717:80738 [0] NCCL INFO Channel 10 : 1[be000] -> 13[57000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,15]:dgx2h0194:79748:80424 [0] NCCL INFO Channel 10 : 15[e5000] -> 3[e7000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,14]:dgx2h0194:79747:80615 [0] NCCL INFO Channel 01 : 14[e0000] -> 4[e2000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,12]:dgx2h0194:79745:80741 [0] NCCL INFO Channel 10 : 12[39000] -> 11[3b000] via P2P/IPC\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:58 2021[1,13]:dgx2h0194:79746:80607 [0] NCCL INFO Channel 08 : 13[57000] -> 9[59000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,1]:dgx2h0194:79717:80738 [0] NCCL INFO Channel 11 : 1[be000] -> 13[57000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,15]:dgx2h0194:79748:80424 [0] NCCL INFO Channel 11 : 15[e5000] -> 3[e7000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,14]:dgx2h0194:79747:80615 [0] NCCL INFO Channel 02 : 14[e0000] -> 4[e2000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,12]:dgx2h0194:79745:80741 [0] NCCL INFO Channel 11 : 12[39000] -> 11[3b000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,9]:dgx2h0194:79742:80744 [0] NCCL INFO Connected all rings\n", - "Mon Sep 6 09:04:58 2021[1,13]:dgx2h0194:79746:80607 [0] NCCL INFO Channel 09 : 13[57000] -> 9[59000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,14]:dgx2h0194:79747:80615 [0] NCCL INFO Channel 03 : 14[e0000] -> 4[e2000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,13]:dgx2h0194:79746:80607 [0] NCCL INFO Channel 10 : 13[57000] -> 9[59000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,9]:dgx2h0194:79742:80744 [0] NCCL INFO Channel 00 : 9[59000] -> 10[5e000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,14]:dgx2h0194:79747:80615 [0] NCCL INFO Channel 04 : 14[e0000] -> 4[e2000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,9]:dgx2h0194:79742:80744 [0] NCCL INFO Channel 01 : 9[59000] -> 10[5e000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,13]:dgx2h0194:79746:80607 [0] NCCL INFO Channel 11 : 13[57000] -> 9[59000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,14]:dgx2h0194:79747:80615 [0] NCCL INFO Channel 05 : 14[e0000] -> 4[e2000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,9]:dgx2h0194:79742:80744 [0] NCCL INFO Channel 02 : 9[59000] -> 10[5e000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,2]:dgx2h0194:79722:80271 [0] NCCL INFO Connected all rings\n", - "Mon Sep 6 09:04:58 2021[1,14]:dgx2h0194:79747:80615 [0] NCCL INFO Channel 06 : 14[e0000] -> 4[e2000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,9]:dgx2h0194:79742:80744 [0] NCCL INFO Channel 03 : 9[59000] -> 10[5e000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,2]:dgx2h0194:79722:80271 [0] NCCL INFO Channel 00 : 2[b7000] -> 5[bc000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,14]:dgx2h0194:79747:80615 [0] NCCL INFO Channel 07 : 14[e0000] -> 4[e2000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,9]:dgx2h0194:79742:80744 [0] NCCL INFO Channel 04 : 9[59000] -> 10[5e000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,2]:dgx2h0194:79722:80271 [0] NCCL INFO Channel 01 : 2[b7000] -> 5[bc000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,14]:dgx2h0194:79747:80615 [0] NCCL INFO Channel 08 : 14[e0000] -> 4[e2000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,9]:dgx2h0194:79742:80744 [0] NCCL INFO Channel 05 : 9[59000] -> 10[5e000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,2]:dgx2h0194:79722:80271 [0] NCCL INFO Channel 02 : 2[b7000] -> 5[bc000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,0]:dgx2h0194:79712:80584 [0] NCCL INFO Connected all rings\n", - "Mon Sep 6 09:04:58 2021[1,14]:dgx2h0194:79747:80615 [0] NCCL INFO Channel 09 : 14[e0000] -> 4[e2000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,9]:dgx2h0194:79742:80744 [0] NCCL INFO Channel 06 : 9[59000] -> 10[5e000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,2]:dgx2h0194:79722:80271 [0] NCCL INFO Channel 03 : 2[b7000] -> 5[bc000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,14]:dgx2h0194:79747:80615 [0] NCCL INFO Channel 10 : 14[e0000] -> 4[e2000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,9]:dgx2h0194:79742:80744 [0] NCCL INFO Channel 07 : 9[59000] -> 10[5e000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,2]:dgx2h0194:79722:80271 [0] NCCL INFO Channel 04 : 2[b7000] -> 5[bc000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,14]:dgx2h0194:79747:80615 [0] NCCL INFO Channel 11 : 14[e0000] -> 4[e2000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,9]:dgx2h0194:79742:80744 [0] NCCL INFO Channel 08 : 9[59000] -> 10[5e000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,2]:dgx2h0194:79722:80271 [0] NCCL INFO Channel 05 : 2[b7000] -> 5[bc000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,8]:dgx2h0194:79741:80747 [0] NCCL INFO Connected all rings\n", - "Mon Sep 6 09:04:58 2021[1,9]:dgx2h0194:79742:80744 [0] NCCL INFO Channel 09 : 9[59000] -> 10[5e000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,2]:dgx2h0194:79722:80271 [0] NCCL INFO Channel 06 : 2[b7000] -> 5[bc000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,8]:dgx2h0194:79741:80747 [0] NCCL INFO Channel 00 : 8[34000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,9]:dgx2h0194:79742:80744 [0] NCCL INFO Channel 10 : 9[59000] -> 10[5e000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,2]:dgx2h0194:79722:80271 [0] NCCL INFO Channel 07 : 2[b7000] -> 5[bc000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,8]:dgx2h0194:79741:80747 [0] NCCL INFO Channel 01 : 8[34000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,9]:dgx2h0194:79742:80744 [0] NCCL INFO Channel 11 : 9[59000] -> 10[5e000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,2]:dgx2h0194:79722:80271 [0] NCCL INFO Channel 08 : 2[b7000] -> 5[bc000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,8]:dgx2h0194:79741:80747 [0] NCCL INFO Channel 02 : 8[34000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,3]:dgx2h0194:79727:80587 [0] NCCL INFO Connected all rings\n", - "Mon Sep 6 09:04:58 2021[1,2]:dgx2h0194:79722:80271 [0] NCCL INFO Channel 09 : 2[b7000] -> 5[bc000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,8]:dgx2h0194:79741:80747 [0] NCCL INFO Channel 03 : 8[34000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,2]:dgx2h0194:79722:80271 [0] NCCL INFO Channel 10 : 2[b7000] -> 5[bc000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,8]:dgx2h0194:79741:80747 [0] NCCL INFO Channel 04 : 8[34000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,2]:dgx2h0194:79722:80271 [0] NCCL INFO Channel 11 : 2[b7000] -> 5[bc000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,8]:dgx2h0194:79741:80747 [0] NCCL INFO Channel 05 : 8[34000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,14]:dgx2h0194:79747:80615 [0] NCCL INFO Connected all trees\n", - "Mon Sep 6 09:04:58 2021[1,14]:dgx2h0194:79747:80615 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 8/8/512\n", - "Mon Sep 6 09:04:58 2021[1,8]:dgx2h0194:79741:80747 [0] NCCL INFO Channel 06 : 8[34000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,4]:dgx2h0194:79732:80427 [0] NCCL INFO Channel 00 : 4[e2000] -> 15[e5000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,8]:dgx2h0194:79741:80747 [0] NCCL INFO Channel 07 : 8[34000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,14]:dgx2h0194:79747:80615 [0] NCCL INFO 12 coll channels, 16 p2p channels, 16 p2p channels per peer\n", - "Mon Sep 6 09:04:58 2021[1,4]:dgx2h0194:79732:80427 [0] NCCL INFO Channel 01 : 4[e2000] -> 15[e5000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,8]:dgx2h0194:79741:80747 [0] NCCL INFO Channel 08 : 8[34000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,4]:dgx2h0194:79732:80427 [0] NCCL INFO Channel 02 : 4[e2000] -> 15[e5000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,8]:dgx2h0194:79741:80747 [0] NCCL INFO Channel 09 : 8[34000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,4]:dgx2h0194:79732:80427 [0] NCCL INFO Channel 03 : 4[e2000] -> 15[e5000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,8]:dgx2h0194:79741:80747 [0] NCCL INFO Channel 10 : 8[34000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,10]:dgx2h0194:79743:80602 [0] NCCL INFO Channel 00 : 10[5e000] -> 6[5c000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,4]:dgx2h0194:79732:80427 [0] NCCL INFO Channel 04 : 4[e2000] -> 15[e5000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,10]:dgx2h0194:79743:80602 [0] NCCL INFO Channel 01 : 10[5e000] -> 6[5c000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,8]:dgx2h0194:79741:80747 [0] NCCL INFO Channel 11 : 8[34000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,4]:dgx2h0194:79732:80427 [0] NCCL INFO Channel 05 : 4[e2000] -> 15[e5000] via P2P/IPC\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:58 2021[1,10]:dgx2h0194:79743:80602 [0] NCCL INFO Channel 02 : 10[5e000] -> 6[5c000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,4]:dgx2h0194:79732:80427 [0] NCCL INFO Channel 06 : 4[e2000] -> 15[e5000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,10]:dgx2h0194:79743:80602 [0] NCCL INFO Channel 03 : 10[5e000] -> 6[5c000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,5]:dgx2h0194:79738:80437 [0] NCCL INFO Channel 00 : 5[bc000] -> 1[be000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,4]:dgx2h0194:79732:80427 [0] NCCL INFO Channel 07 : 4[e2000] -> 15[e5000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,10]:dgx2h0194:79743:80602 [0] NCCL INFO Channel 04 : 10[5e000] -> 6[5c000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,5]:dgx2h0194:79738:80437 [0] NCCL INFO Channel 01 : 5[bc000] -> 1[be000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,4]:dgx2h0194:79732:80427 [0] NCCL INFO Channel 08 : 4[e2000] -> 15[e5000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,10]:dgx2h0194:79743:80602 [0] NCCL INFO Channel 05 : 10[5e000] -> 6[5c000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,3]:dgx2h0194:79727:80587 [0] NCCL INFO Channel 00 : 3[e7000] -> 7[b9000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,5]:dgx2h0194:79738:80437 [0] NCCL INFO Channel 02 : 5[bc000] -> 1[be000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,4]:dgx2h0194:79732:80427 [0] NCCL INFO Channel 09 : 4[e2000] -> 15[e5000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,10]:dgx2h0194:79743:80602 [0] NCCL INFO Channel 06 : 10[5e000] -> 6[5c000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,3]:dgx2h0194:79727:80587 [0] NCCL INFO Channel 01 : 3[e7000] -> 7[b9000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,5]:dgx2h0194:79738:80437 [0] NCCL INFO Channel 03 : 5[bc000] -> 1[be000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,4]:dgx2h0194:79732:80427 [0] NCCL INFO Channel 10 : 4[e2000] -> 15[e5000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,10]:dgx2h0194:79743:80602 [0] NCCL INFO Channel 07 : 10[5e000] -> 6[5c000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,5]:dgx2h0194:79738:80437 [0] NCCL INFO Channel 04 : 5[bc000] -> 1[be000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,3]:dgx2h0194:79727:80587 [0] NCCL INFO Channel 02 : 3[e7000] -> 7[b9000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,4]:dgx2h0194:79732:80427 [0] NCCL INFO Channel 11 : 4[e2000] -> 15[e5000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,10]:dgx2h0194:79743:80602 [0] NCCL INFO Channel 08 : 10[5e000] -> 6[5c000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,5]:dgx2h0194:79738:80437 [0] NCCL INFO Channel 05 : 5[bc000] -> 1[be000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,3]:dgx2h0194:79727:80587 [0] NCCL INFO Channel 03 : 3[e7000] -> 7[b9000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,0]:dgx2h0194:79712:80584 [0] NCCL INFO Connected all trees\n", - "Mon Sep 6 09:04:58 2021[1,0]:dgx2h0194:79712:80584 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 8/8/512\n", - "Mon Sep 6 09:04:58 2021[1,10]:dgx2h0194:79743:80602 [0] NCCL INFO Channel 09 : 10[5e000] -> 6[5c000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,5]:dgx2h0194:79738:80437 [0] NCCL INFO Channel 06 : 5[bc000] -> 1[be000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,3]:dgx2h0194:79727:80587 [0] NCCL INFO Channel 04 : 3[e7000] -> 7[b9000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,0]:dgx2h0194:79712:80584 [0] NCCL INFO 12 coll channels, 16 p2p channels, 16 p2p channels per peer\n", - "Mon Sep 6 09:04:58 2021[1,10]:dgx2h0194:79743:80602 [0] NCCL INFO Channel 10 : 10[5e000] -> 6[5c000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,5]:dgx2h0194:79738:80437 [0] NCCL INFO Channel 07 : 5[bc000] -> 1[be000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,3]:dgx2h0194:79727:80587 [0] NCCL INFO Channel 05 : 3[e7000] -> 7[b9000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,10]:dgx2h0194:79743:80602 [0] NCCL INFO Channel 11 : 10[5e000] -> 6[5c000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,5]:dgx2h0194:79738:80437 [0] NCCL INFO Channel 08 : 5[bc000] -> 1[be000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,3]:dgx2h0194:79727:80587 [0] NCCL INFO Channel 06 : 3[e7000] -> 7[b9000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,5]:dgx2h0194:79738:80437 [0] NCCL INFO Channel 09 : 5[bc000] -> 1[be000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,3]:dgx2h0194:79727:80587 [0] NCCL INFO Channel 07 : 3[e7000] -> 7[b9000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,5]:dgx2h0194:79738:80437 [0] NCCL INFO Channel 10 : 5[bc000] -> 1[be000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,3]:dgx2h0194:79727:80587 [0] NCCL INFO Channel 08 : 3[e7000] -> 7[b9000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,5]:dgx2h0194:79738:80437 [0] NCCL INFO Channel 11 : 5[bc000] -> 1[be000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,3]:dgx2h0194:79727:80587 [0] NCCL INFO Channel 09 : 3[e7000] -> 7[b9000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,3]:dgx2h0194:79727:80587 [0] NCCL INFO Channel 10 : 3[e7000] -> 7[b9000] via P2P/IPC\n", - "Mon Sep 6 09:04:58 2021[1,3]:dgx2h0194:79727:80587 [0] NCCL INFO Channel 11 : 3[e7000] -> 7[b9000] via P2P/IPC\n", - "Mon Sep 6 09:04:59 2021[1,9]:dgx2h0194:79742:80744 [0] NCCL INFO Connected all trees\n", - "Mon Sep 6 09:04:59 2021[1,9]:dgx2h0194:79742:80744 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 8/8/512\n", - "Mon Sep 6 09:04:59 2021[1,6]:dgx2h0194:79739:80608 [0] NCCL INFO Connected all trees\n", - "Mon Sep 6 09:04:59 2021[1,6]:dgx2h0194:79739:80608 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 8/8/512\n", - "Mon Sep 6 09:04:59 2021[1,10]:dgx2h0194:79743:80602 [0] NCCL INFO Connected all trees\n", - "Mon Sep 6 09:04:59 2021[1,10]:dgx2h0194:79743:80602 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 8/8/512\n", - "Mon Sep 6 09:04:59 2021[1,9]:dgx2h0194:79742:80744 [0] NCCL INFO 12 coll channels, 16 p2p channels, 16 p2p channels per peer\n", - "Mon Sep 6 09:04:59 2021[1,6]:dgx2h0194:79739:80608 [0] NCCL INFO 12 coll channels, 16 p2p channels, 16 p2p channels per peer\n", - "Mon Sep 6 09:04:59 2021[1,10]:dgx2h0194:79743:80602 [0] NCCL INFO 12 coll channels, 16 p2p channels, 16 p2p channels per peer\n", - "Mon Sep 6 09:04:59 2021[1,5]:dgx2h0194:79738:80437 [0] NCCL INFO Connected all trees\n", - "Mon Sep 6 09:04:59 2021[1,5]:dgx2h0194:79738:80437 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 8/8/512\n", - "Mon Sep 6 09:04:59 2021[1,5]:dgx2h0194:79738:80437 [0] NCCL INFO 12 coll channels, 16 p2p channels, 16 p2p channels per peer\n", - "Mon Sep 6 09:04:59 2021[1,7]:dgx2h0194:79740:80605 [0] NCCL INFO Channel 00 : 7[b9000] -> 2[b7000] via P2P/IPC\n", - "Mon Sep 6 09:04:59 2021[1,13]:dgx2h0194:79746:80607 [0] NCCL INFO Connected all trees\n", - "Mon Sep 6 09:04:59 2021[1,13]:dgx2h0194:79746:80607 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 8/8/512\n", - "Mon Sep 6 09:04:59 2021[1,7]:dgx2h0194:79740:80605 [0] NCCL INFO Channel 01 : 7[b9000] -> 2[b7000] via P2P/IPC\n", - "Mon Sep 6 09:04:59 2021[1,7]:dgx2h0194:79740:80605 [0] NCCL INFO Channel 02 : 7[b9000] -> 2[b7000] via P2P/IPC\n", - "Mon Sep 6 09:04:59 2021[1,13]:dgx2h0194:79746:80607 [0] NCCL INFO 12 coll channels, 16 p2p channels, 16 p2p channels per peer\n", - "Mon Sep 6 09:04:59 2021[1,7]:dgx2h0194:79740:80605 [0] NCCL INFO Channel 03 : 7[b9000] -> 2[b7000] via P2P/IPC\n", - "Mon Sep 6 09:04:59 2021[1,8]:dgx2h0194:79741:80747 [0] NCCL INFO Connected all trees\n", - "Mon Sep 6 09:04:59 2021[1,8]:dgx2h0194:79741:80747 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 8/8/512\n", - "Mon Sep 6 09:04:59 2021[1,7]:dgx2h0194:79740:80605 [0] NCCL INFO Channel 04 : 7[b9000] -> 2[b7000] via P2P/IPC\n", - "Mon Sep 6 09:04:59 2021[1,1]:dgx2h0194:79717:80738 [0] NCCL INFO Connected all trees\n", - "Mon Sep 6 09:04:59 2021[1,1]:dgx2h0194:79717:80738 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 8/8/512\n", - "Mon Sep 6 09:04:59 2021[1,8]:dgx2h0194:79741:80747 [0] NCCL INFO 12 coll channels, 16 p2p channels, 16 p2p channels per peer\n", - "Mon Sep 6 09:04:59 2021[1,7]:dgx2h0194:79740:80605 [0] NCCL INFO Channel 05 : 7[b9000] -> 2[b7000] via P2P/IPC\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:04:59 2021[1,1]:dgx2h0194:79717:80738 [0] NCCL INFO 12 coll channels, 16 p2p channels, 16 p2p channels per peer\n", - "Mon Sep 6 09:04:59 2021[1,7]:dgx2h0194:79740:80605 [0] NCCL INFO Channel 06 : 7[b9000] -> 2[b7000] via P2P/IPC\n", - "Mon Sep 6 09:04:59 2021[1,3]:dgx2h0194:79727:80587 [0] NCCL INFO Connected all trees\n", - "Mon Sep 6 09:04:59 2021[1,3]:dgx2h0194:79727:80587 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 8/8/512\n", - "Mon Sep 6 09:04:59 2021[1,7]:dgx2h0194:79740:80605 [0] NCCL INFO Channel 07 : 7[b9000] -> 2[b7000] via P2P/IPC\n", - "Mon Sep 6 09:04:59 2021[1,7]:dgx2h0194:79740:80605 [0] NCCL INFO Channel 08 : 7[b9000] -> 2[b7000] via P2P/IPC\n", - "Mon Sep 6 09:04:59 2021[1,3]:dgx2h0194:79727:80587 [0] NCCL INFO 12 coll channels, 16 p2p channels, 16 p2p channels per peer\n", - "Mon Sep 6 09:04:59 2021[1,7]:dgx2h0194:79740:80605 [0] NCCL INFO Channel 09 : 7[b9000] -> 2[b7000] via P2P/IPC\n", - "Mon Sep 6 09:04:59 2021[1,7]:dgx2h0194:79740:80605 [0] NCCL INFO Channel 10 : 7[b9000] -> 2[b7000] via P2P/IPC\n", - "Mon Sep 6 09:04:59 2021[1,7]:dgx2h0194:79740:80605 [0] NCCL INFO Channel 11 : 7[b9000] -> 2[b7000] via P2P/IPC\n", - "Mon Sep 6 09:04:59 2021[1,15]:dgx2h0194:79748:80424 [0] NCCL INFO Connected all trees\n", - "Mon Sep 6 09:04:59 2021[1,15]:dgx2h0194:79748:80424 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 8/8/512\n", - "Mon Sep 6 09:04:59 2021[1,2]:dgx2h0194:79722:80271 [0] NCCL INFO Connected all trees\n", - "Mon Sep 6 09:04:59 2021[1,2]:dgx2h0194:79722:80271 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 8/8/512\n", - "Mon Sep 6 09:04:59 2021[1,7]:dgx2h0194:79740:80605 [0] NCCL INFO Connected all trees\n", - "Mon Sep 6 09:04:59 2021[1,7]:dgx2h0194:79740:80605 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 8/8/512\n", - "Mon Sep 6 09:04:59 2021[1,4]:dgx2h0194:79732:80427 [0] NCCL INFO Connected all trees\n", - "Mon Sep 6 09:04:59 2021[1,4]:dgx2h0194:79732:80427 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 8/8/512\n", - "Mon Sep 6 09:04:59 2021[1,15]:dgx2h0194:79748:80424 [0] NCCL INFO 12 coll channels, 16 p2p channels, 16 p2p channels per peer\n", - "Mon Sep 6 09:04:59 2021[1,2]:dgx2h0194:79722:80271 [0] NCCL INFO 12 coll channels, 16 p2p channels, 16 p2p channels per peer\n", - "Mon Sep 6 09:04:59 2021[1,7]:dgx2h0194:79740:80605 [0] NCCL INFO 12 coll channels, 16 p2p channels, 16 p2p channels per peer\n", - "Mon Sep 6 09:04:59 2021[1,4]:dgx2h0194:79732:80427 [0] NCCL INFO 12 coll channels, 16 p2p channels, 16 p2p channels per peer\n", - "Mon Sep 6 09:04:59 2021[1,11]:dgx2h0194:79744:80434 [0] NCCL INFO Connected all trees\n", - "Mon Sep 6 09:04:59 2021[1,11]:dgx2h0194:79744:80434 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 8/8/512\n", - "Mon Sep 6 09:04:59 2021[1,12]:dgx2h0194:79745:80741 [0] NCCL INFO Connected all trees\n", - "Mon Sep 6 09:04:59 2021[1,12]:dgx2h0194:79745:80741 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 8/8/512\n", - "Mon Sep 6 09:04:59 2021[1,11]:dgx2h0194:79744:80434 [0] NCCL INFO 12 coll channels, 16 p2p channels, 16 p2p channels per peer\n", - "Mon Sep 6 09:04:59 2021[1,12]:dgx2h0194:79745:80741 [0] NCCL INFO 12 coll channels, 16 p2p channels, 16 p2p channels per peer\n", - "Mon Sep 6 09:04:59 2021[1,4]:dgx2h0194:79732:80427 [0] NCCL INFO comm 0x7f159c62d200 rank 4 nranks 16 cudaDev 0 busId e2000 - Init COMPLETE\n", - "Mon Sep 6 09:04:59 2021[1,7]:dgx2h0194:79740:80605 [0] NCCL INFO comm 0x7fec6462ce60 rank 7 nranks 16 cudaDev 0 busId b9000 - Init COMPLETE\n", - "Mon Sep 6 09:04:59 2021[1,15]:dgx2h0194:79748:80424 [0] NCCL INFO comm 0x7fbe3862ca40 rank 15 nranks 16 cudaDev 0 busId e5000 - Init COMPLETE\n", - "Mon Sep 6 09:04:59 2021[1,12]:dgx2h0194:79745:80741 [0] NCCL INFO comm 0x7f00e462e120 rank 12 nranks 16 cudaDev 0 busId 39000 - Init COMPLETE\n", - "Mon Sep 6 09:04:59 2021[1,2]:dgx2h0194:79722:80271 [0] NCCL INFO comm 0x7f9c0062da40 rank 2 nranks 16 cudaDev 0 busId b7000 - Init COMPLETE\n", - "Mon Sep 6 09:04:59 2021[1,8]:dgx2h0194:79741:80747 [0] NCCL INFO comm 0x7f3e2462d9e0 rank 8 nranks 16 cudaDev 0 busId 34000 - Init COMPLETE\n", - "Mon Sep 6 09:04:59 2021[1,10]:dgx2h0194:79743:80602 [0] NCCL INFO comm 0x7f50ac62cfb0 rank 10 nranks 16 cudaDev 0 busId 5e000 - Init COMPLETE\n", - "Mon Sep 6 09:04:59 2021[1,11]:dgx2h0194:79744:80434 [0] NCCL INFO comm 0x7f47c062cad0 rank 11 nranks 16 cudaDev 0 busId 3b000 - Init COMPLETE\n", - "Mon Sep 6 09:04:59 2021[1,5]:dgx2h0194:79738:80437 [0] NCCL INFO comm 0x7f7c2c62d050 rank 5 nranks 16 cudaDev 0 busId bc000 - Init COMPLETE\n", - "Mon Sep 6 09:04:59 2021[1,6]:dgx2h0194:79739:80608 [0] NCCL INFO comm 0x7f10a862d190 rank 6 nranks 16 cudaDev 0 busId 5c000 - Init COMPLETE\n", - "Mon Sep 6 09:04:59 2021[1,1]:dgx2h0194:79717:80738 [0] NCCL INFO comm 0x7f635462cf70 rank 1 nranks 16 cudaDev 0 busId be000 - Init COMPLETE\n", - "Mon Sep 6 09:04:59 2021[1,0]:dgx2h0194:79712:80584 [0] NCCL INFO comm 0x7fa9b065f3d0 rank 0 nranks 16 cudaDev 0 busId 36000 - Init COMPLETE\n", - "Mon Sep 6 09:04:59 2021[1,9]:dgx2h0194:79742:80744 [0] NCCL INFO comm 0x7f330c62cf00 rank 9 nranks 16 cudaDev 0 busId 59000 - Init COMPLETE\n", - "Mon Sep 6 09:04:59 2021[1,3]:dgx2h0194:79727:80587 [0] NCCL INFO comm 0x7f786062ca30 rank 3 nranks 16 cudaDev 0 busId e7000 - Init COMPLETE\n", - "Mon Sep 6 09:04:59 2021[1,13]:dgx2h0194:79746:80607 [0] NCCL INFO comm 0x7ff78062ce20 rank 13 nranks 16 cudaDev 0 busId 57000 - Init COMPLETE\n", - "Mon Sep 6 09:04:59 2021[1,14]:dgx2h0194:79747:80615 [0] NCCL INFO comm 0x7fa09462d170 rank 14 nranks 16 cudaDev 0 busId e0000 - Init COMPLETE\n", - "Mon Sep 6 09:04:59 2021[1,0]:dgx2h0194:79712:80584 [0] NCCL INFO Launch mode Parallel\n", - "Mon Sep 6 09:04:59 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:05:08 2021[1,3]:2021-09-06 09:05:08.906524: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 09:05:08 2021[1,3]:2021-09-06 09:05:08.906568: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 09:05:08 2021[1,9]:2021-09-06 09:05:08.907713: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 09:05:08 2021[1,9]:2021-09-06 09:05:08.907738: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 09:05:08 2021[1,11]:2021-09-06 09:05:08.915421: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 09:05:08 2021[1,11]:2021-09-06 09:05:08.915453: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 09:05:08 2021[1,1]:2021-09-06 09:05:08.920370: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 09:05:08 2021[1,1]:2021-09-06 09:05:08.920406: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 09:05:08 2021[1,15]:2021-09-06 09:05:08.931629: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 09:05:08 2021[1,15]:2021-09-06 09:05:08.931668: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 09:05:08 2021[1,0]:2021-09-06 09:05:08.935354: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 09:05:08 2021[1,0]:2021-09-06 09:05:08.935382: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 09:05:08 2021[1,4]:2021-09-06 09:05:08.941341: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 09:05:08 2021[1,4]:2021-09-06 09:05:08.941368: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 09:05:08 2021[1,6]:2021-09-06 09:05:08.945179: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 09:05:08 2021[1,6]:2021-09-06 09:05:08.945222: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 09:05:08 2021[1,12]:2021-09-06 09:05:08.953442: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 09:05:08 2021[1,12]:2021-09-06 09:05:08.953478: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 09:05:08 2021[1,5]:2021-09-06 09:05:08.955782: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 09:05:08 2021[1,5]:2021-09-06 09:05:08.955817: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 09:05:08 2021[1,13]:2021-09-06 09:05:08.955878: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 09:05:08 2021[1,13]:2021-09-06 09:05:08.955902: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 09:05:08 2021[1,2]:2021-09-06 09:05:08.964502: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 09:05:08 2021[1,2]:2021-09-06 09:05:08.964533: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 09:05:08 2021[1,10]:2021-09-06 09:05:08.966637: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 09:05:08 2021[1,10]:2021-09-06 09:05:08.966668: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 09:05:08 2021[1,7]:2021-09-06 09:05:08.975908: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 09:05:08 2021[1,7]:2021-09-06 09:05:08.975936: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 09:05:08 2021[1,8]:2021-09-06 09:05:08.977191: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 09:05:08 2021[1,8]:2021-09-06 09:05:08.977212: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 09:05:08 2021[1,14]:2021-09-06 09:05:08.977981: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 09:05:08 2021[1,14]:2021-09-06 09:05:08.978007: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 09:05:10 2021[1,15]:2021-09-06 09:05:10.305228: I tensorflow/core/profiler/lib/profiler_session.cc:66] Profiler session collecting data.\n", - "Mon Sep 6 09:05:10 2021[1,2]:2021-09-06 09:05:10.305433: I tensorflow/core/profiler/lib/profiler_session.cc:66] Profiler session collecting data.\n", - "Mon Sep 6 09:05:10 2021[1,4]:2021-09-06 09:05:10.305459: I tensorflow/core/profiler/lib/profiler_session.cc:66] Profiler session collecting data.\n", - "Mon Sep 6 09:05:10 2021[1,5]:2021-09-06 09:05:10.305543: I tensorflow/core/profiler/lib/profiler_session.cc:66] Profiler session collecting data.\n", - "Mon Sep 6 09:05:10 2021[1,15]:2021-09-06 09:05:10.306514: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 09:05:10 2021[1,2]:2021-09-06 09:05:10.306618: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 09:05:10 2021[1,5]:2021-09-06 09:05:10.306708: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 09:05:10 2021[1,4]:2021-09-06 09:05:10.307508: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 09:05:10 2021[1,13]:2021-09-06 09:05:10.309214: I tensorflow/core/profiler/lib/profiler_session.cc:66] Profiler session collecting data.\n", - "Mon Sep 6 09:05:10 2021[1,6]:2021-09-06 09:05:10.309215: I tensorflow/core/profiler/lib/profiler_session.cc:66] Profiler session collecting data.\n", - "Mon Sep 6 09:05:10 2021[1,7]:2021-09-06 09:05:10.309278: I tensorflow/core/profiler/lib/profiler_session.cc:66] Profiler session collecting data.\n", - "Mon Sep 6 09:05:10 2021[1,9]:2021-09-06 09:05:10.309264: I tensorflow/core/profiler/lib/profiler_session.cc:66] Profiler session collecting data.\n", - "Mon Sep 6 09:05:10 2021[1,10]:2021-09-06 09:05:10.309370: I tensorflow/core/profiler/lib/profiler_session.cc:66] Profiler session collecting data.\n", - "Mon Sep 6 09:05:10 2021[1,13]:2021-09-06 09:05:10.310410: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 09:05:10 2021[1,7]:2021-09-06 09:05:10.310457: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 09:05:10 2021[1,9]:2021-09-06 09:05:10.310481: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 09:05:10 2021[1,10]:2021-09-06 09:05:10.310655: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 09:05:10 2021[1,6]:2021-09-06 09:05:10.311490: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 09:05:10 2021[1,3]:2021-09-06 09:05:10.316781: I tensorflow/core/profiler/lib/profiler_session.cc:66] Profiler session collecting data.\n", - "Mon Sep 6 09:05:10 2021[1,3]:2021-09-06 09:05:10.319146: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - " 1/4000 [..............................]Mon Sep 6 09:04:59 2021[1,0]: - ETA: 31:27:06 - loss: 0.9112 - auc: 0.4892Mon Sep 6 09:05:10 2021[1,0]:\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:05:10 2021[1,11]:2021-09-06 09:05:10.342218: I tensorflow/core/profiler/lib/profiler_session.cc:66] Profiler session collecting data.\n", - "Mon Sep 6 09:05:10 2021[1,8]:2021-09-06 09:05:10.342267: I tensorflow/core/profiler/lib/profiler_session.cc:66] Profiler session collecting data.\n", - "Mon Sep 6 09:05:10 2021[1,1]:2021-09-06 09:05:10.342284: I tensorflow/core/profiler/lib/profiler_session.cc:66] Profiler session collecting data.\n", - "Mon Sep 6 09:05:10 2021[1,14]:2021-09-06 09:05:10.342547: I tensorflow/core/profiler/lib/profiler_session.cc:66] Profiler session collecting data.\n", - "Mon Sep 6 09:05:10 2021[1,0]:2021-09-06 09:05:10.342667: I tensorflow/core/profiler/lib/profiler_session.cc:66] Profiler session collecting data.\n", - "Mon Sep 6 09:05:10 2021[1,11]:2021-09-06 09:05:10.343662: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 09:05:10 2021[1,1]:2021-09-06 09:05:10.343704: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 09:05:10 2021[1,0]:2021-09-06 09:05:10.344116: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 09:05:10 2021[1,8]:2021-09-06 09:05:10.344609: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 09:05:10 2021[1,12]:2021-09-06 09:05:10.345052: I tensorflow/core/profiler/lib/profiler_session.cc:66] Profiler session collecting data.\n", - "Mon Sep 6 09:05:10 2021[1,14]:2021-09-06 09:05:10.345700: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 09:05:10 2021[1,12]:2021-09-06 09:05:10.346509: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 09:05:10 2021[1,2]:2021-09-06 09:05:10.358042: I tensorflow/core/profiler/internal/gpu/cupti_collector.cc:673] GpuTracer has collected 2649 callback api events and 2583 activity events. \n", - "Mon Sep 6 09:05:10 2021[1,13]:2021-09-06 09:05:10.358854: I tensorflow/core/profiler/internal/gpu/cupti_collector.cc:673] GpuTracer has collected 2649 callback api events and 2583 activity events. \n", - "Mon Sep 6 09:05:10 2021[1,7]:2021-09-06 09:05:10.359228: I tensorflow/core/profiler/internal/gpu/cupti_collector.cc:673] GpuTracer has collected 2648 callback api events and 2582 activity events. \n", - "Mon Sep 6 09:05:10 2021[1,9]:2021-09-06 09:05:10.360356: I tensorflow/core/profiler/internal/gpu/cupti_collector.cc:673] GpuTracer has collected 2648 callback api events and 2582 activity events. \n", - "Mon Sep 6 09:05:10 2021[1,4]:2021-09-06 09:05:10.364826: I tensorflow/core/profiler/internal/gpu/cupti_collector.cc:673] GpuTracer has collected 2648 callback api events and 2582 activity events. \n", - "Mon Sep 6 09:05:10 2021[1,15]:2021-09-06 09:05:10.367892: I tensorflow/core/profiler/internal/gpu/cupti_collector.cc:673] GpuTracer has collected 2648 callback api events and 2582 activity events. \n", - "Mon Sep 6 09:05:10 2021[1,10]:2021-09-06 09:05:10.368670: I tensorflow/core/profiler/internal/gpu/cupti_collector.cc:673] GpuTracer has collected 2652 callback api events and 2586 activity events. \n", - "Mon Sep 6 09:05:10 2021[1,6]:2021-09-06 09:05:10.369775: I tensorflow/core/profiler/internal/gpu/cupti_collector.cc:673] GpuTracer has collected 2649 callback api events and 2583 activity events. \n", - "Mon Sep 6 09:05:10 2021[1,5]:2021-09-06 09:05:10.373559: I tensorflow/core/profiler/internal/gpu/cupti_collector.cc:673] GpuTracer has collected 2651 callback api events and 2585 activity events. \n", - "Mon Sep 6 09:05:10 2021[1,3]:2021-09-06 09:05:10.383506: I tensorflow/core/profiler/internal/gpu/cupti_collector.cc:673] GpuTracer has collected 2648 callback api events and 2582 activity events. \n", - "Mon Sep 6 09:05:10 2021[1,0]:2021-09-06 09:05:10.392125: I tensorflow/core/profiler/internal/gpu/cupti_collector.cc:673] GpuTracer has collected 2648 callback api events and 2582 activity events. \n", - "Mon Sep 6 09:05:10 2021[1,11]:2021-09-06 09:05:10.392648: I tensorflow/core/profiler/internal/gpu/cupti_collector.cc:673] GpuTracer has collected 2648 callback api events and 2582 activity events. \n", - "Mon Sep 6 09:05:10 2021[1,12]:2021-09-06 09:05:10.393439: I tensorflow/core/profiler/internal/gpu/cupti_collector.cc:673] GpuTracer has collected 2648 callback api events and 2582 activity events. \n", - "Mon Sep 6 09:05:10 2021[1,1]:2021-09-06 09:05:10.393559: I tensorflow/core/profiler/internal/gpu/cupti_collector.cc:673] GpuTracer has collected 2648 callback api events and 2582 activity events. \n", - "Mon Sep 6 09:05:10 2021[1,2]:2021-09-06 09:05:10.400431: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 09:05:10 2021[1,13]:2021-09-06 09:05:10.402509: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 09:05:10 2021[1,7]:2021-09-06 09:05:10.402749: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 09:05:10 2021[1,8]:2021-09-06 09:05:10.405715: I tensorflow/core/profiler/internal/gpu/cupti_collector.cc:673] GpuTracer has collected 2648 callback api events and 2582 activity events. \n", - "Mon Sep 6 09:05:10 2021[1,9]:2021-09-06 09:05:10.410689: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 09:05:10 2021[1,6]:2021-09-06 09:05:10.415497: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 09:05:10 2021[1,15]:2021-09-06 09:05:10.415594: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 09:05:10 2021[1,5]:2021-09-06 09:05:10.415980: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 09:05:10 2021[1,4]:2021-09-06 09:05:10.419823: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 09:05:10 2021[1,10]:2021-09-06 09:05:10.424162: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 09:05:10 2021[1,3]:2021-09-06 09:05:10.426899: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 09:05:10 2021[1,0]:2021-09-06 09:05:10.436674: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 09:05:10 2021[1,12]:2021-09-06 09:05:10.437051: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 09:05:10 2021[1,14]:2021-09-06 09:05:10.437544: I tensorflow/core/profiler/internal/gpu/cupti_collector.cc:673] GpuTracer has collected 2648 callback api events and 2582 activity events. \n", - "Mon Sep 6 09:05:10 2021[1,1]:2021-09-06 09:05:10.437594: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 09:05:10 2021[1,11]:2021-09-06 09:05:10.440492: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 09:05:10 2021[1,2]:2021-09-06 09:05:10.452744: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,2]:\n", - "Mon Sep 6 09:05:10 2021[1,8]:2021-09-06 09:05:10.453531: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 09:05:10 2021[1,13]:2021-09-06 09:05:10.454222: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,13]:\n", - "Mon Sep 6 09:05:10 2021[1,7]:2021-09-06 09:05:10.454482: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,7]:\n", - "Mon Sep 6 09:05:10 2021[1,5]:2021-09-06 09:05:10.467085: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:05:10 2021[1,5]:\n", - "Mon Sep 6 09:05:10 2021[1,9]:2021-09-06 09:05:10.468653: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,9]:\n", - "Mon Sep 6 09:05:10 2021[1,6]:2021-09-06 09:05:10.469858: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,6]:\n", - "Mon Sep 6 09:05:10 2021[1,15]:2021-09-06 09:05:10.471366: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,15]:\n", - "Mon Sep 6 09:05:10 2021[1,4]:2021-09-06 09:05:10.477411: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,4]:\n", - "Mon Sep 6 09:05:10 2021[1,14]:2021-09-06 09:05:10.478317: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 09:05:10 2021[1,10]:2021-09-06 09:05:10.479776: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,10]:\n", - "Mon Sep 6 09:05:10 2021[1,2]:2021-09-06 09:05:10.480068: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for trace.json.gz to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.trace.json.gz\n", - "Mon Sep 6 09:05:10 2021[1,13]:2021-09-06 09:05:10.481862: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for trace.json.gz to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.trace.json.gz\n", - "Mon Sep 6 09:05:10 2021[1,7]:2021-09-06 09:05:10.482391: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for trace.json.gz to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.trace.json.gz\n", - "Mon Sep 6 09:05:10 2021[1,3]:2021-09-06 09:05:10.483605: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,3]:\n", - "Mon Sep 6 09:05:10 2021[1,12]:2021-09-06 09:05:10.485892: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,12]:\n", - "Mon Sep 6 09:05:10 2021[1,1]:2021-09-06 09:05:10.488111: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,1]:\n", - "Mon Sep 6 09:05:10 2021[1,0]:2021-09-06 09:05:10.488549: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,0]:\n", - "Mon Sep 6 09:05:10 2021[1,11]:2021-09-06 09:05:10.492599: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,11]:\n", - "Mon Sep 6 09:05:10 2021[1,5]:2021-09-06 09:05:10.495745: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for trace.json.gz to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.trace.json.gz\n", - "Mon Sep 6 09:05:10 2021[1,9]:2021-09-06 09:05:10.497173: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for trace.json.gz to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.trace.json.gz\n", - "Mon Sep 6 09:05:10 2021[1,6]:2021-09-06 09:05:10.498271: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for trace.json.gz to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.trace.json.gz\n", - "Mon Sep 6 09:05:10 2021[1,15]:2021-09-06 09:05:10.502046: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for trace.json.gz to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.trace.json.gz\n", - "Mon Sep 6 09:05:10 2021[1,8]:2021-09-06 09:05:10.505250: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,8]:\n", - "Mon Sep 6 09:05:10 2021[1,4]:2021-09-06 09:05:10.505717: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for trace.json.gz to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.trace.json.gz\n", - "Mon Sep 6 09:05:10 2021[1,10]:2021-09-06 09:05:10.510133: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for trace.json.gz to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.trace.json.gz\n", - "Mon Sep 6 09:05:10 2021[1,3]:2021-09-06 09:05:10.511769: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for trace.json.gz to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.trace.json.gz\n", - "Mon Sep 6 09:05:10 2021[1,12]:2021-09-06 09:05:10.513653: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for trace.json.gz to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.trace.json.gz\n", - "Mon Sep 6 09:05:10 2021[1,1]:2021-09-06 09:05:10.516357: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for trace.json.gz to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.trace.json.gz\n", - "Mon Sep 6 09:05:10 2021[1,0]:2021-09-06 09:05:10.516749: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for trace.json.gz to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.trace.json.gz\n", - "Mon Sep 6 09:05:10 2021[1,11]:2021-09-06 09:05:10.520974: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for trace.json.gz to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.trace.json.gz\n", - "Mon Sep 6 09:05:10 2021[1,14]:2021-09-06 09:05:10.533211: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,14]:\n", - "Mon Sep 6 09:05:10 2021[1,8]:2021-09-06 09:05:10.533268: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for trace.json.gz to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.trace.json.gz\n", - "Mon Sep 6 09:05:10 2021[1,2]:2021-09-06 09:05:10.543716: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,2]:\n", - "Mon Sep 6 09:05:10 2021[1,13]:2021-09-06 09:05:10.544599: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,13]:\n", - "Mon Sep 6 09:05:10 2021[1,7]:2021-09-06 09:05:10.545527: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,7]:\n", - "Mon Sep 6 09:05:10 2021[1,2]:2021-09-06 09:05:10.551058: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for memory_profile.json.gz to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.memory_profile.json.gz\n", - "Mon Sep 6 09:05:10 2021[1,13]:2021-09-06 09:05:10.551684: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for memory_profile.json.gz to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.memory_profile.json.gz\n", - "Mon Sep 6 09:05:10 2021[1,7]:2021-09-06 09:05:10.552556: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for memory_profile.json.gz to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.memory_profile.json.gz\n", - "Mon Sep 6 09:05:10 2021[1,2]:2021-09-06 09:05:10.554892: I tensorflow/core/profiler/rpc/client/capture_profile.cc:251] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,2]:Dumped tool data for xplane.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.xplane.pb\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:05:10 2021[1,2]:Dumped tool data for overview_page.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.overview_page.pb\n", - "Mon Sep 6 09:05:10 2021[1,2]:Dumped tool data for input_pipeline.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.input_pipeline.pb\n", - "Mon Sep 6 09:05:10 2021[1,2]:Dumped tool data for tensorflow_stats.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.tensorflow_stats.pb\n", - "Mon Sep 6 09:05:10 2021[1,2]:Dumped tool data for kernel_stats.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.kernel_stats.pb\n", - "Mon Sep 6 09:05:10 2021[1,2]:\n", - "Mon Sep 6 09:05:10 2021[1,13]:2021-09-06 09:05:10.555408: I tensorflow/core/profiler/rpc/client/capture_profile.cc:251] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,13]:Dumped tool data for xplane.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.xplane.pb\n", - "Mon Sep 6 09:05:10 2021[1,13]:Dumped tool data for overview_page.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.overview_page.pb\n", - "Mon Sep 6 09:05:10 2021[1,13]:Dumped tool data for input_pipeline.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.input_pipeline.pb\n", - "Mon Sep 6 09:05:10 2021[1,13]:Dumped tool data for tensorflow_stats.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.tensorflow_stats.pb\n", - "Mon Sep 6 09:05:10 2021[1,13]:Dumped tool data for kernel_stats.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.kernel_stats.pb\n", - "Mon Sep 6 09:05:10 2021[1,13]:\n", - "Mon Sep 6 09:05:10 2021[1,7]:2021-09-06 09:05:10.556035: I tensorflow/core/profiler/rpc/client/capture_profile.cc:251] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,7]:Dumped tool data for xplane.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.xplane.pb\n", - "Mon Sep 6 09:05:10 2021[1,7]:Dumped tool data for overview_page.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.overview_page.pb\n", - "Mon Sep 6 09:05:10 2021[1,7]:Dumped tool data for input_pipeline.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.input_pipeline.pb\n", - "Mon Sep 6 09:05:10 2021[1,7]:Dumped tool data for tensorflow_stats.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.tensorflow_stats.pb\n", - "Mon Sep 6 09:05:10 2021[1,7]:Dumped tool data for kernel_stats.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.kernel_stats.pb\n", - "Mon Sep 6 09:05:10 2021[1,7]:\n", - "Mon Sep 6 09:05:10 2021[1,9]:2021-09-06 09:05:10.559516: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,9]:\n", - "Mon Sep 6 09:05:10 2021[1,5]:2021-09-06 09:05:10.560753: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,5]:\n", - "Mon Sep 6 09:05:10 2021[1,6]:2021-09-06 09:05:10.562556: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,6]:\n", - "Mon Sep 6 09:05:10 2021[1,9]:2021-09-06 09:05:10.566678: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for memory_profile.json.gz to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.memory_profile.json.gz\n", - "Mon Sep 6 09:05:10 2021[1,5]:2021-09-06 09:05:10.568086: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for memory_profile.json.gz to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.memory_profile.json.gz\n", - "Mon Sep 6 09:05:10 2021[1,6]:2021-09-06 09:05:10.569781: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for memory_profile.json.gz to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.memory_profile.json.gz\n", - "Mon Sep 6 09:05:10 2021[1,9]:2021-09-06 09:05:10.570665: I tensorflow/core/profiler/rpc/client/capture_profile.cc:251] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,9]:Dumped tool data for xplane.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.xplane.pb\n", - "Mon Sep 6 09:05:10 2021[1,9]:Dumped tool data for overview_page.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.overview_page.pb\n", - "Mon Sep 6 09:05:10 2021[1,9]:Dumped tool data for input_pipeline.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.input_pipeline.pb\n", - "Mon Sep 6 09:05:10 2021[1,9]:Dumped tool data for tensorflow_stats.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.tensorflow_stats.pb\n", - "Mon Sep 6 09:05:10 2021[1,9]:Dumped tool data for kernel_stats.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.kernel_stats.pb\n", - "Mon Sep 6 09:05:10 2021[1,9]:\n", - "Mon Sep 6 09:05:10 2021[1,14]:2021-09-06 09:05:10.572285: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for trace.json.gz to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.trace.json.gz\n", - "Mon Sep 6 09:05:10 2021[1,5]:2021-09-06 09:05:10.572728: I tensorflow/core/profiler/rpc/client/capture_profile.cc:251] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,5]:Dumped tool data for xplane.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.xplane.pb\n", - "Mon Sep 6 09:05:10 2021[1,5]:Dumped tool data for overview_page.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.overview_page.pb\n", - "Mon Sep 6 09:05:10 2021[1,5]:Dumped tool data for input_pipeline.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.input_pipeline.pb\n", - "Mon Sep 6 09:05:10 2021[1,5]:Dumped tool data for tensorflow_stats.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.tensorflow_stats.pb\n", - "Mon Sep 6 09:05:10 2021[1,5]:Dumped tool data for kernel_stats.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.kernel_stats.pb\n", - "Mon Sep 6 09:05:10 2021[1,5]:\n", - "Mon Sep 6 09:05:10 2021[1,6]:2021-09-06 09:05:10.573924: I tensorflow/core/profiler/rpc/client/capture_profile.cc:251] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,6]:Dumped tool data for xplane.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.xplane.pb\n", - "Mon Sep 6 09:05:10 2021[1,6]:Dumped tool data for overview_page.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.overview_page.pb\n", - "Mon Sep 6 09:05:10 2021[1,6]:Dumped tool data for input_pipeline.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.input_pipeline.pb\n", - "Mon Sep 6 09:05:10 2021[1,6]:Dumped tool data for tensorflow_stats.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.tensorflow_stats.pb\n", - "Mon Sep 6 09:05:10 2021[1,6]:Dumped tool data for kernel_stats.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.kernel_stats.pb\n", - "Mon Sep 6 09:05:10 2021[1,6]:\n", - "Mon Sep 6 09:05:10 2021[1,12]:2021-09-06 09:05:10.578011: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,12]:\n", - "Mon Sep 6 09:05:10 2021[1,3]:2021-09-06 09:05:10.580067: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,3]:\n", - "Mon Sep 6 09:05:10 2021[1,4]:2021-09-06 09:05:10.581250: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,4]:\n", - "Mon Sep 6 09:05:10 2021[1,0]:2021-09-06 09:05:10.581588: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,0]:\n", - "Mon Sep 6 09:05:10 2021[1,12]:2021-09-06 09:05:10.585279: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for memory_profile.json.gz to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.memory_profile.json.gz\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:05:10 2021[1,11]:2021-09-06 09:05:10.585794: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,11]:\n", - "Mon Sep 6 09:05:10 2021[1,15]:2021-09-06 09:05:10.586581: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,15]:\n", - "Mon Sep 6 09:05:10 2021[1,3]:2021-09-06 09:05:10.587215: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for memory_profile.json.gz to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.memory_profile.json.gz\n", - "Mon Sep 6 09:05:10 2021[1,12]:2021-09-06 09:05:10.589198: I tensorflow/core/profiler/rpc/client/capture_profile.cc:251] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,12]:Dumped tool data for xplane.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.xplane.pb\n", - "Mon Sep 6 09:05:10 2021[1,12]:Dumped tool data for overview_page.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.overview_page.pb\n", - "Mon Sep 6 09:05:10 2021[1,12]:Dumped tool data for input_pipeline.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.input_pipeline.pb\n", - "Mon Sep 6 09:05:10 2021[1,12]:Dumped tool data for tensorflow_stats.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.tensorflow_stats.pb\n", - "Mon Sep 6 09:05:10 2021[1,12]:Dumped tool data for kernel_stats.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.kernel_stats.pb\n", - "Mon Sep 6 09:05:10 2021[1,12]:\n", - "Mon Sep 6 09:05:10 2021[1,0]:2021-09-06 09:05:10.589550: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for memory_profile.json.gz to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.memory_profile.json.gz\n", - "Mon Sep 6 09:05:10 2021[1,4]:2021-09-06 09:05:10.590763: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for memory_profile.json.gz to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.memory_profile.json.gz\n", - "Mon Sep 6 09:05:10 2021[1,3]:2021-09-06 09:05:10.591167: I tensorflow/core/profiler/rpc/client/capture_profile.cc:251] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,3]:Dumped tool data for xplane.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.xplane.pb\n", - "Mon Sep 6 09:05:10 2021[1,3]:Dumped tool data for overview_page.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.overview_page.pb\n", - "Mon Sep 6 09:05:10 2021[1,3]:Dumped tool data for input_pipeline.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.input_pipeline.pb\n", - "Mon Sep 6 09:05:10 2021[1,3]:Dumped tool data for tensorflow_stats.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.tensorflow_stats.pb\n", - "Mon Sep 6 09:05:10 2021[1,3]:Dumped tool data for kernel_stats.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.kernel_stats.pb\n", - "Mon Sep 6 09:05:10 2021[1,3]:\n", - "Mon Sep 6 09:05:10 2021[1,11]:2021-09-06 09:05:10.593091: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for memory_profile.json.gz to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.memory_profile.json.gz\n", - "Mon Sep 6 09:05:10 2021[1,0]:2021-09-06 09:05:10.594288: I tensorflow/core/profiler/rpc/client/capture_profile.cc:251] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,0]:Dumped tool data for xplane.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.xplane.pb\n", - "Mon Sep 6 09:05:10 2021[1,0]:Dumped tool data for overview_page.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.overview_page.pb\n", - "Mon Sep 6 09:05:10 2021[1,0]:Dumped tool data for input_pipeline.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.input_pipeline.pb\n", - "Mon Sep 6 09:05:10 2021[1,0]:Dumped tool data for tensorflow_stats.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.tensorflow_stats.pb\n", - "Mon Sep 6 09:05:10 2021[1,0]:Dumped tool data for kernel_stats.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.kernel_stats.pb\n", - "Mon Sep 6 09:05:10 2021[1,0]:\n", - "Mon Sep 6 09:05:10 2021[1,15]:2021-09-06 09:05:10.596368: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for memory_profile.json.gz to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.memory_profile.json.gz\n", - "Mon Sep 6 09:05:10 2021[1,4]:2021-09-06 09:05:10.596671: I tensorflow/core/profiler/rpc/client/capture_profile.cc:251] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,4]:Dumped tool data for xplane.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.xplane.pb\n", - "Mon Sep 6 09:05:10 2021[1,4]:Dumped tool data for overview_page.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.overview_page.pb\n", - "Mon Sep 6 09:05:10 2021[1,4]:Dumped tool data for input_pipeline.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.input_pipeline.pb\n", - "Mon Sep 6 09:05:10 2021[1,4]:Dumped tool data for tensorflow_stats.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.tensorflow_stats.pb\n", - "Mon Sep 6 09:05:10 2021[1,4]:Dumped tool data for kernel_stats.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.kernel_stats.pb\n", - "Mon Sep 6 09:05:10 2021[1,4]:\n", - "Mon Sep 6 09:05:10 2021[1,11]:2021-09-06 09:05:10.597264: I tensorflow/core/profiler/rpc/client/capture_profile.cc:251] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,11]:Dumped tool data for xplane.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.xplane.pb\n", - "Mon Sep 6 09:05:10 2021[1,11]:Dumped tool data for overview_page.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.overview_page.pb\n", - "Mon Sep 6 09:05:10 2021[1,11]:Dumped tool data for input_pipeline.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.input_pipeline.pb\n", - "Mon Sep 6 09:05:10 2021[1,11]:Dumped tool data for tensorflow_stats.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.tensorflow_stats.pb\n", - "Mon Sep 6 09:05:10 2021[1,11]:Dumped tool data for kernel_stats.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.kernel_stats.pb\n", - "Mon Sep 6 09:05:10 2021[1,11]:\n", - "Mon Sep 6 09:05:10 2021[1,10]:2021-09-06 09:05:10.598865: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,10]:\n", - "Mon Sep 6 09:05:10 2021[1,1]:2021-09-06 09:05:10.601237: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,1]:\n", - "Mon Sep 6 09:05:10 2021[1,15]:2021-09-06 09:05:10.601247: I tensorflow/core/profiler/rpc/client/capture_profile.cc:251] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,15]:Dumped tool data for xplane.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.xplane.pb\n", - "Mon Sep 6 09:05:10 2021[1,15]:Dumped tool data for overview_page.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.overview_page.pb\n", - "Mon Sep 6 09:05:10 2021[1,15]:Dumped tool data for input_pipeline.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.input_pipeline.pb\n", - "Mon Sep 6 09:05:10 2021[1,15]:Dumped tool data for tensorflow_stats.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.tensorflow_stats.pb\n", - "Mon Sep 6 09:05:10 2021[1,15]:Dumped tool data for kernel_stats.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.kernel_stats.pb\n", - "Mon Sep 6 09:05:10 2021[1,15]:\n", - "Mon Sep 6 09:05:10 2021[1,8]:2021-09-06 09:05:10.601392: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:05:10 2021[1,8]:\n", - "Mon Sep 6 09:05:10 2021[1,8]:2021-09-06 09:05:10.609086: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for memory_profile.json.gz to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.memory_profile.json.gz\n", - "Mon Sep 6 09:05:10 2021[1,10]:2021-09-06 09:05:10.609124: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for memory_profile.json.gz to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.memory_profile.json.gz\n", - "Mon Sep 6 09:05:10 2021[1,1]:2021-09-06 09:05:10.609444: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for memory_profile.json.gz to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.memory_profile.json.gz\n", - "Mon Sep 6 09:05:10 2021[1,8]:2021-09-06 09:05:10.614234: I tensorflow/core/profiler/rpc/client/capture_profile.cc:251] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,8]:Dumped tool data for xplane.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.xplane.pb\n", - "Mon Sep 6 09:05:10 2021[1,8]:Dumped tool data for overview_page.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.overview_page.pb\n", - "Mon Sep 6 09:05:10 2021[1,8]:Dumped tool data for input_pipeline.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.input_pipeline.pb\n", - "Mon Sep 6 09:05:10 2021[1,8]:Dumped tool data for tensorflow_stats.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.tensorflow_stats.pb\n", - "Mon Sep 6 09:05:10 2021[1,8]:Dumped tool data for kernel_stats.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.kernel_stats.pb\n", - "Mon Sep 6 09:05:10 2021[1,8]:\n", - "Mon Sep 6 09:05:10 2021[1,10]:2021-09-06 09:05:10.614705: I tensorflow/core/profiler/rpc/client/capture_profile.cc:251] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,10]:Dumped tool data for xplane.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.xplane.pb\n", - "Mon Sep 6 09:05:10 2021[1,10]:Dumped tool data for overview_page.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.overview_page.pb\n", - "Mon Sep 6 09:05:10 2021[1,10]:Dumped tool data for input_pipeline.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.input_pipeline.pb\n", - "Mon Sep 6 09:05:10 2021[1,10]:Dumped tool data for tensorflow_stats.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.tensorflow_stats.pb\n", - "Mon Sep 6 09:05:10 2021[1,10]:Dumped tool data for kernel_stats.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.kernel_stats.pb\n", - "Mon Sep 6 09:05:10 2021[1,10]:\n", - "Mon Sep 6 09:05:10 2021[1,1]:2021-09-06 09:05:10.617210: I tensorflow/core/profiler/rpc/client/capture_profile.cc:251] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,1]:Dumped tool data for xplane.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.xplane.pb\n", - "Mon Sep 6 09:05:10 2021[1,1]:Dumped tool data for overview_page.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.overview_page.pb\n", - "Mon Sep 6 09:05:10 2021[1,1]:Dumped tool data for input_pipeline.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.input_pipeline.pb\n", - "Mon Sep 6 09:05:10 2021[1,1]:Dumped tool data for tensorflow_stats.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.tensorflow_stats.pb\n", - "Mon Sep 6 09:05:10 2021[1,1]:Dumped tool data for kernel_stats.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.kernel_stats.pb\n", - "Mon Sep 6 09:05:10 2021[1,1]:\n", - "Mon Sep 6 09:05:10 2021[1,14]:2021-09-06 09:05:10.677166: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,14]:\n", - "Mon Sep 6 09:05:10 2021[1,14]:2021-09-06 09:05:10.684214: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for memory_profile.json.gz to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.memory_profile.json.gz\n", - "Mon Sep 6 09:05:10 2021[1,14]:2021-09-06 09:05:10.687649: I tensorflow/core/profiler/rpc/client/capture_profile.cc:251] Creating directory: tf_logs/plugins/profile/2021_09_06_09_05_10\n", - "Mon Sep 6 09:05:10 2021[1,14]:Dumped tool data for xplane.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.xplane.pb\n", - "Mon Sep 6 09:05:10 2021[1,14]:Dumped tool data for overview_page.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.overview_page.pb\n", - "Mon Sep 6 09:05:10 2021[1,14]:Dumped tool data for input_pipeline.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.input_pipeline.pb\n", - "Mon Sep 6 09:05:10 2021[1,14]:Dumped tool data for tensorflow_stats.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.tensorflow_stats.pb\n", - "Mon Sep 6 09:05:10 2021[1,14]:Dumped tool data for kernel_stats.pb to tf_logs/plugins/profile/2021_09_06_09_05_10/dgx2h0194.kernel_stats.pb\n", - "Mon Sep 6 09:05:10 2021[1,14]:\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - " 5/4000 [..............................]Mon Sep 6 09:05:11 2021[1,0]: - ETA: 3:15:09 - loss: 0.8028 - auc: 0.4982Mon Sep 6 09:05:11 2021[1,0]Mon Sep 6 09:05:10 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:05:11 2021[1,12]:WARNING:tensorflow:Callback method `on_train_batch_begin` is slow compared to the batch time (batch time: 0.1455s vs `on_train_batch_begin` time: 0.1987s). Check your callbacks.\n", - "Mon Sep 6 09:05:11 2021[1,12]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.1455s vs `on_train_batch_end` time: 1.6635s). Check your callbacks.\n", - "Mon Sep 6 09:05:11 2021[1,3]:WARNING:tensorflow:Callback method `on_train_batch_begin` is slow compared to the batch time (batch time: 0.1708s vs `on_train_batch_begin` time: 0.2090s). Check your callbacks.\n", - "Mon Sep 6 09:05:11 2021[1,3]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.1708s vs `on_train_batch_end` time: 1.6582s). Check your callbacks.\n", - "Mon Sep 6 09:05:11 2021[1,10]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.1720s vs `on_train_batch_end` time: 1.6715s). Check your callbacks.\n", - "Mon Sep 6 09:05:11 2021[1,7]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.1905s vs `on_train_batch_end` time: 1.6623s). Check your callbacks.\n", - "Mon Sep 6 09:05:11 2021[1,8]:WARNING:tensorflow:Callback method `on_train_batch_begin` is slow compared to the batch time (batch time: 0.1486s vs `on_train_batch_begin` time: 0.1849s). Check your callbacks.\n", - "Mon Sep 6 09:05:11 2021[1,13]:WARNING:tensorflow:Callback method `on_train_batch_begin` is slow compared to the batch time (batch time: 0.1773s vs `on_train_batch_begin` time: 0.2086s). Check your callbacks.\n", - "Mon Sep 6 09:05:11 2021[1,13]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.1773s vs `on_train_batch_end` time: 1.6576s). Check your callbacks.\n", - "Mon Sep 6 09:05:11 2021[1,5]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.1958s vs `on_train_batch_end` time: 1.6611s). Check your callbacks.\n", - "Mon Sep 6 09:05:11 2021[1,1]:WARNING:tensorflow:Callback method `on_train_batch_begin` is slow compared to the batch time (batch time: 0.1316s vs `on_train_batch_begin` time: 0.2118s). Check your callbacks.\n", - "Mon Sep 6 09:05:11 2021[1,1]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.1316s vs `on_train_batch_end` time: 1.6669s). Check your callbacks.\n", - "Mon Sep 6 09:05:11 2021[1,8]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.1486s vs `on_train_batch_end` time: 1.6714s). Check your callbacks.\n", - "Mon Sep 6 09:05:11 2021[1,4]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.1880s vs `on_train_batch_end` time: 1.6633s). Check your callbacks.\n", - "Mon Sep 6 09:05:11 2021[1,6]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.1824s vs `on_train_batch_end` time: 1.6600s). Check your callbacks.\n", - "Mon Sep 6 09:05:11 2021[1,2]:WARNING:tensorflow:Callback method `on_train_batch_begin` is slow compared to the batch time (batch time: 0.1603s vs `on_train_batch_begin` time: 0.2171s). Check your callbacks.\n", - "Mon Sep 6 09:05:11 2021[1,11]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.1957s vs `on_train_batch_end` time: 1.6587s). Check your callbacks.\n", - "Mon Sep 6 09:05:11 2021[1,2]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.1603s vs `on_train_batch_end` time: 1.6624s). Check your callbacks.\n", - "Mon Sep 6 09:05:11 2021[1,15]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.1938s vs `on_train_batch_end` time: 1.6640s). Check your callbacks.\n", - "Mon Sep 6 09:05:11 2021[1,9]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.1979s vs `on_train_batch_end` time: 1.6555s). Check your callbacks.\n", - "Mon Sep 6 09:05:11 2021[1,14]:WARNING:tensorflow:Callback method `on_train_batch_begin` is slow compared to the batch time (batch time: 0.1465s vs `on_train_batch_begin` time: 0.1744s). Check your callbacks.\n", - "Mon Sep 6 09:05:11 2021[1,14]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.1465s vs `on_train_batch_end` time: 1.6863s). Check your callbacks.\n", - "Mon Sep 6 09:05:11 2021[1,0]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.1925s vs `on_train_batch_end` time: 1.6647s). Check your callbacks.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - " 212/4000 [>.............................]Mon Sep 6 09:05:52 2021[1,0]: - ETA: 15:58 - loss: 0.1657 - auc: 0.6865Mon Sep 6 09:05:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - " 513/4000 [==>...........................]Mon Sep 6 09:06:52 2021[1,0]: - ETA: 12:52 - loss: 0.1432 - auc: 0.7375Mon Sep 6 09:06:52 2021[1,0] (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - " 799/4000 [====>.........................]Mon Sep 6 09:07:50 2021[1,0]: - ETA: 11:25 - loss: 0.1371 - auc: 0.7529Mon Sep 6 09:07:52 2021[1,0] (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1102/4000 [=======>......................]Mon Sep 6 09:08:52 2021[1,0]: - ETA: 10:14 - loss: 0.1339 - auc: 0.7614Mon Sep 6 09:08:52 2021[1,0] (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1398/4000 [=========>....................]Mon Sep 6 09:09:49 2021[1,0]: - ETA: 9:01 - loss: 0.1321 - auc: 0.7666Mon Sep 6 09:09:50 2021[1,0]3Mon Sep 6 09:09:47 2021[1,0]>\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1702/4000 [===========>..................]Mon Sep 6 09:10:52 2021[1,0]: - ETA: 7:57 - loss: 0.1308 - auc: 0.7706Mon Sep 6 09:10:52 2021[1,0]5Mon Sep 6 09:10:52 2021[1,0]Mon Sep 6 09:10:08 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 09:11:50 2021[1,0]:1998/4000 [=============>................] - ETA: 6:52 - loss: 0.1299 - auc: 0.7735Mon Sep 6 09:11:50 2021[1,0]1Mon Sep 6 09:11:42 2021[1,0]Mon Sep 6 09:10:58 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2300/4000 [================>.............]Mon Sep 6 09:12:52 2021[1,0]: - ETA: 5:50 - loss: 0.1292 - auc: 0.7758Mon Sep 6 09:12:52 2021[1,0]8Mon Sep 6 09:12:52 2021[1,0]\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2599/4000 [==================>...........]Mon Sep 6 09:13:50 2021[1,0]: - ETA: 4:46 - loss: 0.1286 - auc: 0.7776Mon Sep 6 09:13:53 2021[1,0]6Mon Sep 6 09:13:50 2021[1,0]\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2910/4000 [====================>.........]Mon Sep 6 09:14:52 2021[1,0]: - ETA: 3:42 - loss: 0.1282 - auc: 0.7792Mon Sep 6 09:14:53 2021[1,0]2Mon Sep 6 09:14:52 2021[1,0]\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3199/4000 [======================>.......]Mon Sep 6 09:15:49 2021[1,0]: - ETA: 2:42 - loss: 0.1278 - auc: 0.7805Mon Sep 6 09:15:51 2021[1,0]5Mon Sep 6 09:15:48 2021[1,0]\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3512/4000 [=========================>....]Mon Sep 6 09:16:52 2021[1,0]: - ETA: 1:39 - loss: 0.1274 - auc: 0.7816Mon Sep 6 09:16:53 2021[1,0]6Mon Sep 6 09:16:52 2021[1,0]\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3799/4000 [===========================>..]Mon Sep 6 09:17:50 2021[1,0]: - ETA: 40s - loss: 0.1272 - auc: 0.7826Mon Sep 6 09:17:52 2021[1,0]\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "4000/4000 [==============================]Mon Sep 6 09:18:36 2021[1,0]: - 845s 204ms/step - loss: 0.1270 - auc: 0.7832 - val_loss: 0.1273 - val_auc: 0.7933 6 09:18:22 2021[1,0]\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - " \r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Best Loss: 0.127242\n" - ] - } - ], - "source": [ - "main()" - ] - }, - { - "cell_type": "markdown", - "id": "5a1c01ad", - "metadata": {}, - "source": [ - "## Use Petastorm\n", - "\n", - "Here we set `--dataloader` to `petastorm` to force Petastorm run." - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "fee60d10", - "metadata": {}, - "outputs": [], - "source": [ - "def main_petastorm():\n", - " parser = argparse.ArgumentParser(description='Criteo Spark Keras Training Example',\n", - " formatter_class=argparse.ArgumentDefaultsHelpFormatter)\n", - " parser.add_argument('--data-dir', default='file:///opt/data/criteo/parquet',\n", - " help='location of the transformed Criteo dataset in Parquet format')\n", - " parser.add_argument('--logs-dir', default='/opt/experiments/criteo', help='location of TensorFlow logs')\n", - " parser.add_argument('--dataloader', default=PETASTORM_DATALOADER,\n", - " choices=[PETASTORM_DATALOADER, NVTABULAR_DATALOADER],\n", - " help='dataloader to use')\n", - " parser.add_argument('--num-proc', type=int, default=1, help='number of worker processes for training')\n", - " parser.add_argument('--learning-rate', type=float, default=0.0001, help='initial learning rate')\n", - " parser.add_argument('--batch-size', type=int, default=64 * 1024, help='batch size')\n", - " parser.add_argument('--epochs', type=int, default=3, help='number of epochs to train')\n", - " parser.add_argument('--local-checkpoint-file', default='checkpoint', help='model checkpoint')\n", - " args = parser.parse_args(args=['--num-proc', '16', '--data-dir', 'file:///raid/spark-team/criteo/parquet', \n", - " '--dataloader', 'petastorm', '--learning-rate', '0.001',\n", - " '--batch-size', '65535','--epochs', '1', '--logs-dir', 'tf_logs',\n", - " '--local-checkpoint-file', 'ckpt_file'])\n", - " \n", - "\n", - " dimensions = get_category_dimensions(spark, args.data_dir)\n", - "\n", - " train_df = spark.read.parquet(f'{args.data_dir}/train')\n", - " val_df = spark.read.parquet(f'{args.data_dir}/val')\n", - " test_df = spark.read.parquet(f'{args.data_dir}/test')\n", - " train_rows, val_rows, test_rows = train_df.count(), val_df.count(), test_df.count()\n", - " print('Training: %d' % train_rows)\n", - " print('Validation: %d' % val_rows)\n", - " print('Test: %d' % test_rows)\n", - "\n", - " train(dimensions, train_rows, val_rows, args)\n", - "\n", - " spark.stop()" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "id": "1cb35049", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "21/09/06 11:44:44 WARN package: Truncated the string representation of a plan since it was too large. This behavior can be adjusted by setting 'spark.sql.debug.maxToStringFields'.\n", - " \r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'c0': '7912888',\n", - " 'c1': '33822',\n", - " 'c10': '582468',\n", - " 'c11': '245827',\n", - " 'c12': '10',\n", - " 'c13': '2208',\n", - " 'c14': '10666',\n", - " 'c15': '103',\n", - " 'c16': '3',\n", - " 'c17': '967',\n", - " 'c18': '14',\n", - " 'c19': '8165895',\n", - " 'c2': '17138',\n", - " 'c20': '2675939',\n", - " 'c21': '7156452',\n", - " 'c22': '302515',\n", - " 'c23': '12021',\n", - " 'c24': '96',\n", - " 'c25': '34',\n", - " 'c3': '7338',\n", - " 'c4': '20045',\n", - " 'c5': '3',\n", - " 'c6': '7104',\n", - " 'c7': '1381',\n", - " 'c8': '62',\n", - " 'c9': '5554113'}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - " \r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Training: 4195197692\n", - "Validation: 89137318\n", - "Test: 89137319\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Checking whether extension tensorflow was built with MPI.\n", - "Extension tensorflow was built with MPI.\n", - "mpirun --allow-run-as-root --tag-output -np 16 -H dgx2h0194-a1adff968d508e8d1142986f3e2c42dc:16 -bind-to none -map-by slot -mca pml ob1 -mca btl ^openib --timestamp-output -mca btl_tcp_if_include enp134s0f0 -x NCCL_IB_GID_INDEX=3 -x NCCL_DEBUG=INFO -mca plm_rsh_agent \"/home/ngc-auth-ldap-allxu/miniconda3/bin/python -m horovod.spark.driver.mpirun_rsh gAWVcAEAAAAAAAB9lCiMAmxvlF2UjAkxMjcuMC4wLjGUTU5+hpRhjAdlbnA1M3MwlF2UjAwxMC4xNDguMzAuNTmUTU5+hpRhjAdlbnA1OHMwlF2UjAwxMC4xNDguOTQuNTmUTU5+hpRhjAdlbnA4OHMwlF2UjAwxMC4xNDkuMzAuMzSUTU5+hpRhjAdlbnA5M3MwlF2UjAwxMC4xNDkuOTQuNTeUTU5+hpRhjAplbnAxMzRzMGYwlF2UjAsxMC4xNTAuMzAuMpRNTn6GlGGMCGVucDE4NHMwlF2UjA0xMC4xNDguMTU4LjU5lE1OfoaUYYwIZW5wMTg5czCUXZSMDTEwLjE0OC4yMjIuNTmUTU5+hpRhjAhlbnAyMjVzMJRdlIwNMTAuMTQ5LjE1OC41N5RNTn6GlGGMCGVucDIzMHMwlF2UjA0xMC4xNDkuMjIyLjU3lE1OfoaUYYwHZG9ja2VyMJRdlIwKMTcyLjE3LjAuMZRNTn6GlGF1Lg== gAWVAwMAAAAAAACMI2hvcm92b2QucnVubmVyLmNvbW1vbi51dGlsLnNldHRpbmdzlIwIU2V0dGluZ3OUk5QpgZR9lCiMCG51bV9wcm9jlEsQjAd2ZXJib3NllEsCjAhzc2hfcG9ydJROjBFzc2hfaWRlbnRpdHlfZmlsZZROjA5leHRyYV9tcGlfYXJnc5SMOS1tY2EgYnRsX3RjcF9pZl9pbmNsdWRlIGVucDEzNHMwZjAgLXggTkNDTF9JQl9HSURfSU5ERVg9M5SMCHRjcF9mbGFnlE6MDGJpbmRpbmdfYXJnc5ROjANrZXmUTowNc3RhcnRfdGltZW91dJSMImhvcm92b2QucnVubmVyLmNvbW1vbi51dGlsLnRpbWVvdXSUjAdUaW1lb3V0lJOUKYGUfZQojAhfdGltZW91dJRNWAKMC190aW1lb3V0X2F0lEdB2E2AYtTXh4wIX21lc3NhZ2WUWA4BAABUaW1lZCBvdXQgd2FpdGluZyBmb3Ige2FjdGl2aXR5fS4gUGxlYXNlIGNoZWNrIHRoYXQgeW91IGhhdmUgZW5vdWdoIHJlc291cmNlcyB0byBydW4gYWxsIEhvcm92b2QgcHJvY2Vzc2VzLiBFYWNoIEhvcm92b2QgcHJvY2VzcyBydW5zIGluIGEgU3BhcmsgdGFzay4gWW91IG1heSBuZWVkIHRvIGluY3JlYXNlIHRoZSBzdGFydF90aW1lb3V0IHBhcmFtZXRlciB0byBhIGxhcmdlciB2YWx1ZSBpZiB5b3VyIFNwYXJrIHJlc291cmNlcyBhcmUgYWxsb2NhdGVkIG9uLWRlbWFuZC6UdWKMD291dHB1dF9maWxlbmFtZZROjA1ydW5fZnVuY19tb2RllIiMBG5pY3OUfZSMB2VsYXN0aWOUiYwccHJlZml4X291dHB1dF93aXRoX3RpbWVzdGFtcJSIjAVob3N0c5SMLWRneDJoMDE5NC1hMWFkZmY5NjhkNTA4ZThkMTE0Mjk4NmYzZTJjNDJkYzoxNpR1Yi4=\" /home/ngc-auth-ldap-allxu/miniconda3/bin/python -m horovod.spark.task.mpirun_exec_fn gAWVcAEAAAAAAAB9lCiMAmxvlF2UjAkxMjcuMC4wLjGUTU5+hpRhjAdlbnA1M3MwlF2UjAwxMC4xNDguMzAuNTmUTU5+hpRhjAdlbnA1OHMwlF2UjAwxMC4xNDguOTQuNTmUTU5+hpRhjAdlbnA4OHMwlF2UjAwxMC4xNDkuMzAuMzSUTU5+hpRhjAdlbnA5M3MwlF2UjAwxMC4xNDkuOTQuNTeUTU5+hpRhjAplbnAxMzRzMGYwlF2UjAsxMC4xNTAuMzAuMpRNTn6GlGGMCGVucDE4NHMwlF2UjA0xMC4xNDguMTU4LjU5lE1OfoaUYYwIZW5wMTg5czCUXZSMDTEwLjE0OC4yMjIuNTmUTU5+hpRhjAhlbnAyMjVzMJRdlIwNMTAuMTQ5LjE1OC41N5RNTn6GlGGMCGVucDIzMHMwlF2UjA0xMC4xNDkuMjIyLjU3lE1OfoaUYYwHZG9ja2VyMJRdlIwKMTcyLjE3LjAuMZRNTn6GlGF1Lg== gAWVkwkAAAAAAACMI2hvcm92b2QucnVubmVyLmNvbW1vbi51dGlsLnNldHRpbmdzlIwIU2V0dGluZ3OUk5QpgZR9lCiMCG51bV9wcm9jlEsQjAd2ZXJib3NllEsCjAhzc2hfcG9ydJROjBFzc2hfaWRlbnRpdHlfZmlsZZROjA5leHRyYV9tcGlfYXJnc5RYxgYAAC1tY2EgYnRsX3RjcF9pZl9pbmNsdWRlIGVucDEzNHMwZjAgLXggTkNDTF9JQl9HSURfSU5ERVg9MyAteCBOQ0NMX0RFQlVHPUlORk8gLW1jYSBwbG1fcnNoX2FnZW50ICIvaG9tZS9uZ2MtYXV0aC1sZGFwLWFsbHh1L21pbmljb25kYTMvYmluL3B5dGhvbiAtbSBob3Jvdm9kLnNwYXJrLmRyaXZlci5tcGlydW5fcnNoIGdBV1ZjQUVBQUFBQUFBQjlsQ2lNQW14dmxGMlVqQWt4TWpjdU1DNHdMakdVVFU1K2hwUmhqQWRsYm5BMU0zTXdsRjJVakF3eE1DNHhORGd1TXpBdU5UbVVUVTUraHBSaGpBZGxibkExT0hNd2xGMlVqQXd4TUM0eE5EZ3VPVFF1TlRtVVRVNStocFJoakFkbGJuQTRPSE13bEYyVWpBd3hNQzR4TkRrdU16QXVNelNVVFU1K2hwUmhqQWRsYm5BNU0zTXdsRjJVakF3eE1DNHhORGt1T1RRdU5UZVVUVTUraHBSaGpBcGxibkF4TXpSek1HWXdsRjJVakFzeE1DNHhOVEF1TXpBdU1wUk5UbjZHbEdHTUNHVnVjREU0TkhNd2xGMlVqQTB4TUM0eE5EZ3VNVFU0TGpVNWxFMU9mb2FVWVl3SVpXNXdNVGc1Y3pDVVhaU01EVEV3TGpFME9DNHlNakl1TlRtVVRVNStocFJoakFobGJuQXlNalZ6TUpSZGxJd05NVEF1TVRRNUxqRTFPQzQxTjVSTlRuNkdsR0dNQ0dWdWNESXpNSE13bEYyVWpBMHhNQzR4TkRrdU1qSXlMalUzbEUxT2ZvYVVZWXdIWkc5amEyVnlNSlJkbEl3S01UY3lMakUzTGpBdU1aUk5UbjZHbEdGMUxnPT0gZ0FXVkF3TUFBQUFBQUFDTUkyaHZjbTkyYjJRdWNuVnVibVZ5TG1OdmJXMXZiaTUxZEdsc0xuTmxkSFJwYm1kemxJd0lVMlYwZEdsdVozT1VrNVFwZ1pSOWxDaU1DRzUxYlY5d2NtOWpsRXNRakFkMlpYSmliM05sbEVzQ2pBaHpjMmhmY0c5eWRKUk9qQkZ6YzJoZmFXUmxiblJwZEhsZlptbHNaWlJPakE1bGVIUnlZVjl0Y0dsZllYSm5jNVNNT1MxdFkyRWdZblJzWDNSamNGOXBabDlwYm1Oc2RXUmxJR1Z1Y0RFek5ITXdaakFnTFhnZ1RrTkRURjlKUWw5SFNVUmZTVTVFUlZnOU01U01DSFJqY0Y5bWJHRm5sRTZNREdKcGJtUnBibWRmWVhKbmM1Uk9qQU5yWlhtVVRvd05jM1JoY25SZmRHbHRaVzkxZEpTTUltaHZjbTkyYjJRdWNuVnVibVZ5TG1OdmJXMXZiaTUxZEdsc0xuUnBiV1Z2ZFhTVWpBZFVhVzFsYjNWMGxKT1VLWUdVZlpRb2pBaGZkR2x0Wlc5MWRKUk5XQUtNQzE5MGFXMWxiM1YwWDJGMGxFZEIyRTJBWXRUWGg0d0lYMjFsYzNOaFoyV1VXQTRCQUFCVWFXMWxaQ0J2ZFhRZ2QyRnBkR2x1WnlCbWIzSWdlMkZqZEdsMmFYUjVmUzRnVUd4bFlYTmxJR05vWldOcklIUm9ZWFFnZVc5MUlHaGhkbVVnWlc1dmRXZG9JSEpsYzI5MWNtTmxjeUIwYnlCeWRXNGdZV3hzSUVodmNtOTJiMlFnY0hKdlkyVnpjMlZ6TGlCRllXTm9JRWh2Y205MmIyUWdjSEp2WTJWemN5QnlkVzV6SUdsdUlHRWdVM0JoY21zZ2RHRnpheTRnV1c5MUlHMWhlU0J1WldWa0lIUnZJR2x1WTNKbFlYTmxJSFJvWlNCemRHRnlkRjkwYVcxbGIzVjBJSEJoY21GdFpYUmxjaUIwYnlCaElHeGhjbWRsY2lCMllXeDFaU0JwWmlCNWIzVnlJRk53WVhKcklISmxjMjkxY21ObGN5QmhjbVVnWVd4c2IyTmhkR1ZrSUc5dUxXUmxiV0Z1WkM2VWRXS01EMjkxZEhCMWRGOW1hV3hsYm1GdFpaUk9qQTF5ZFc1ZlpuVnVZMTl0YjJSbGxJaU1CRzVwWTNPVWZaU01CMlZzWVhOMGFXT1VpWXdjY0hKbFptbDRYMjkxZEhCMWRGOTNhWFJvWDNScGJXVnpkR0Z0Y0pTSWpBVm9iM04wYzVTTUxXUm5lREpvTURFNU5DMWhNV0ZrWm1ZNU5qaGtOVEE0WlRoa01URTBNams0Tm1ZelpUSmpOREprWXpveE5wUjFZaTQ9IpSMCHRjcF9mbGFnlE6MDGJpbmRpbmdfYXJnc5ROjANrZXmUTowNc3RhcnRfdGltZW91dJSMImhvcm92b2QucnVubmVyLmNvbW1vbi51dGlsLnRpbWVvdXSUjAdUaW1lb3V0lJOUKYGUfZQojAhfdGltZW91dJRNWAKMC190aW1lb3V0X2F0lEdB2E2AYtTXh4wIX21lc3NhZ2WUWA4BAABUaW1lZCBvdXQgd2FpdGluZyBmb3Ige2FjdGl2aXR5fS4gUGxlYXNlIGNoZWNrIHRoYXQgeW91IGhhdmUgZW5vdWdoIHJlc291cmNlcyB0byBydW4gYWxsIEhvcm92b2QgcHJvY2Vzc2VzLiBFYWNoIEhvcm92b2QgcHJvY2VzcyBydW5zIGluIGEgU3BhcmsgdGFzay4gWW91IG1heSBuZWVkIHRvIGluY3JlYXNlIHRoZSBzdGFydF90aW1lb3V0IHBhcmFtZXRlciB0byBhIGxhcmdlciB2YWx1ZSBpZiB5b3VyIFNwYXJrIHJlc291cmNlcyBhcmUgYWxsb2NhdGVkIG9uLWRlbWFuZC6UdWKMD291dHB1dF9maWxlbmFtZZROjA1ydW5fZnVuY19tb2RllIiMBG5pY3OUfZSMB2VsYXN0aWOUiYwccHJlZml4X291dHB1dF93aXRoX3RpbWVzdGFtcJSIjAVob3N0c5SMLWRneDJoMDE5NC1hMWFkZmY5NjhkNTA4ZThkMTE0Mjk4NmYzZTJjNDJkYzoxNpR1Yi4=\n", - "Mon Sep 6 11:44:58 2021[1,5]:Changing cwd from /home/ngc-auth-ldap-allxu to /raid/spark-team/allen-dlrm/spark-3.1.2-bin-hadoop3.2/work/app-20210906114426-0004/1\n", - "Mon Sep 6 11:44:58 2021[1,4]:Changing cwd from /home/ngc-auth-ldap-allxu to /raid/spark-team/allen-dlrm/spark-3.1.2-bin-hadoop3.2/work/app-20210906114426-0004/1\n", - "Mon Sep 6 11:44:58 2021[1,1]:Changing cwd from /home/ngc-auth-ldap-allxu to /raid/spark-team/allen-dlrm/spark-3.1.2-bin-hadoop3.2/work/app-20210906114426-0004/1\n", - "Mon Sep 6 11:44:58 2021[1,0]:Changing cwd from /home/ngc-auth-ldap-allxu to /raid/spark-team/allen-dlrm/spark-3.1.2-bin-hadoop3.2/work/app-20210906114426-0004/1\n", - "Mon Sep 6 11:44:58 2021[1,7]:Changing cwd from /home/ngc-auth-ldap-allxu to /raid/spark-team/allen-dlrm/spark-3.1.2-bin-hadoop3.2/work/app-20210906114426-0004/1\n", - "Mon Sep 6 11:44:58 2021[1,2]:Changing cwd from /home/ngc-auth-ldap-allxu to /raid/spark-team/allen-dlrm/spark-3.1.2-bin-hadoop3.2/work/app-20210906114426-0004/1\n", - "Mon Sep 6 11:44:58 2021[1,3]:Changing cwd from /home/ngc-auth-ldap-allxu to /raid/spark-team/allen-dlrm/spark-3.1.2-bin-hadoop3.2/work/app-20210906114426-0004/1\n", - "Mon Sep 6 11:44:58 2021[1,8]:Changing cwd from /home/ngc-auth-ldap-allxu to /raid/spark-team/allen-dlrm/spark-3.1.2-bin-hadoop3.2/work/app-20210906114426-0004/1\n", - "Mon Sep 6 11:44:58 2021[1,10]:Changing cwd from /home/ngc-auth-ldap-allxu to /raid/spark-team/allen-dlrm/spark-3.1.2-bin-hadoop3.2/work/app-20210906114426-0004/1\n", - "Mon Sep 6 11:44:58 2021[1,11]:Changing cwd from /home/ngc-auth-ldap-allxu to /raid/spark-team/allen-dlrm/spark-3.1.2-bin-hadoop3.2/work/app-20210906114426-0004/1\n", - "Mon Sep 6 11:44:58 2021[1,14]:Changing cwd from /home/ngc-auth-ldap-allxu to /raid/spark-team/allen-dlrm/spark-3.1.2-bin-hadoop3.2/work/app-20210906114426-0004/1\n", - "Mon Sep 6 11:44:58 2021[1,9]:Changing cwd from /home/ngc-auth-ldap-allxu to /raid/spark-team/allen-dlrm/spark-3.1.2-bin-hadoop3.2/work/app-20210906114426-0004/1\n", - "Mon Sep 6 11:44:58 2021[1,6]:Changing cwd from /home/ngc-auth-ldap-allxu to /raid/spark-team/allen-dlrm/spark-3.1.2-bin-hadoop3.2/work/app-20210906114426-0004/1\n", - "Mon Sep 6 11:44:58 2021[1,15]:Changing cwd from /home/ngc-auth-ldap-allxu to /raid/spark-team/allen-dlrm/spark-3.1.2-bin-hadoop3.2/work/app-20210906114426-0004/1\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:44:58 2021[1,12]:Changing cwd from /home/ngc-auth-ldap-allxu to /raid/spark-team/allen-dlrm/spark-3.1.2-bin-hadoop3.2/work/app-20210906114426-0004/1\n", - "Mon Sep 6 11:44:58 2021[1,13]:Changing cwd from /home/ngc-auth-ldap-allxu to /raid/spark-team/allen-dlrm/spark-3.1.2-bin-hadoop3.2/work/app-20210906114426-0004/1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:45:19 2021[1,14]:2021-09-06 11:45:19.644980: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\n", - "Mon Sep 6 11:45:19 2021[1,14]:To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "Mon Sep 6 11:45:19 2021[1,5]:2021-09-06 11:45:19.658879: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\n", - "Mon Sep 6 11:45:19 2021[1,5]:To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "Mon Sep 6 11:45:19 2021[1,2]:2021-09-06 11:45:19.677330: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\n", - "Mon Sep 6 11:45:19 2021[1,2]:To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "Mon Sep 6 11:45:19 2021[1,15]:2021-09-06 11:45:19.692565: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\n", - "Mon Sep 6 11:45:19 2021[1,15]:To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "Mon Sep 6 11:45:19 2021[1,13]:2021-09-06 11:45:19.710165: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\n", - "Mon Sep 6 11:45:19 2021[1,13]:To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "Mon Sep 6 11:45:19 2021[1,11]:2021-09-06 11:45:19.730864: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\n", - "Mon Sep 6 11:45:19 2021[1,11]:To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "Mon Sep 6 11:45:19 2021[1,10]:2021-09-06 11:45:19.755246: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\n", - "Mon Sep 6 11:45:19 2021[1,10]:To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "Mon Sep 6 11:45:19 2021[1,7]:2021-09-06 11:45:19.777504: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\n", - "Mon Sep 6 11:45:19 2021[1,7]:To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "Mon Sep 6 11:45:19 2021[1,6]:2021-09-06 11:45:19.803092: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\n", - "Mon Sep 6 11:45:19 2021[1,6]:To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "Mon Sep 6 11:45:19 2021[1,3]:2021-09-06 11:45:19.823747: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\n", - "Mon Sep 6 11:45:19 2021[1,3]:To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "Mon Sep 6 11:45:19 2021[1,8]:2021-09-06 11:45:19.845900: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\n", - "Mon Sep 6 11:45:19 2021[1,8]:To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "Mon Sep 6 11:45:19 2021[1,4]:2021-09-06 11:45:19.868706: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\n", - "Mon Sep 6 11:45:19 2021[1,4]:To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "Mon Sep 6 11:45:19 2021[1,0]:2021-09-06 11:45:19.955887: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\n", - "Mon Sep 6 11:45:19 2021[1,0]:To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "Mon Sep 6 11:45:20 2021[1,9]:2021-09-06 11:45:20.026533: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\n", - "Mon Sep 6 11:45:20 2021[1,9]:To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "Mon Sep 6 11:45:20 2021[1,1]:2021-09-06 11:45:20.142189: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\n", - "Mon Sep 6 11:45:20 2021[1,1]:To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "Mon Sep 6 11:45:20 2021[1,12]:2021-09-06 11:45:20.356066: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\n", - "Mon Sep 6 11:45:20 2021[1,12]:To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "Mon Sep 6 11:45:22 2021[1,5]:2021-09-06 11:45:22.866987: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1504] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 30964 MB memory: -> device: 0, name: Tesla V100-SXM3-32GB-H, pci bus id: 0000:59:00.0, compute capability: 7.0\n", - "Mon Sep 6 11:45:22 2021[1,2]:2021-09-06 11:45:22.934896: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1504] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 30964 MB memory: -> device: 0, name: Tesla V100-SXM3-32GB-H, pci bus id: 0000:e5:00.0, compute capability: 7.0\n", - "Mon Sep 6 11:45:22 2021[1,14]:2021-09-06 11:45:22.935986: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1504] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 30964 MB memory: -> device: 0, name: Tesla V100-SXM3-32GB-H, pci bus id: 0000:3b:00.0, compute capability: 7.0\n", - "Mon Sep 6 11:45:23 2021[1,11]:2021-09-06 11:45:23.037281: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1504] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 30964 MB memory: -> device: 0, name: Tesla V100-SXM3-32GB-H, pci bus id: 0000:5e:00.0, compute capability: 7.0\n", - "Mon Sep 6 11:45:23 2021[1,13]:2021-09-06 11:45:23.046456: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1504] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 30964 MB memory: -> device: 0, name: Tesla V100-SXM3-32GB-H, pci bus id: 0000:34:00.0, compute capability: 7.0\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:45:23 2021[1,7]:2021-09-06 11:45:23.107462: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1504] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 30964 MB memory: -> device: 0, name: Tesla V100-SXM3-32GB-H, pci bus id: 0000:5c:00.0, compute capability: 7.0\n", - "Mon Sep 6 11:45:23 2021[1,6]:2021-09-06 11:45:23.108097: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1504] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 30964 MB memory: -> device: 0, name: Tesla V100-SXM3-32GB-H, pci bus id: 0000:b7:00.0, compute capability: 7.0\n", - "Mon Sep 6 11:45:23 2021[1,10]:2021-09-06 11:45:23.112989: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1504] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 30964 MB memory: -> device: 0, name: Tesla V100-SXM3-32GB-H, pci bus id: 0000:e0:00.0, compute capability: 7.0\n", - "Mon Sep 6 11:45:23 2021[1,3]:2021-09-06 11:45:23.203766: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1504] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 30964 MB memory: -> device: 0, name: Tesla V100-SXM3-32GB-H, pci bus id: 0000:e2:00.0, compute capability: 7.0\n", - "Mon Sep 6 11:45:23 2021[1,8]:2021-09-06 11:45:23.211519: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1504] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 30964 MB memory: -> device: 0, name: Tesla V100-SXM3-32GB-H, pci bus id: 0000:be:00.0, compute capability: 7.0\n", - "Mon Sep 6 11:45:23 2021[1,15]:2021-09-06 11:45:23.270928: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1504] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 30964 MB memory: -> device: 0, name: Tesla V100-SXM3-32GB-H, pci bus id: 0000:b9:00.0, compute capability: 7.0\n", - "Mon Sep 6 11:45:23 2021[1,4]:2021-09-06 11:45:23.279986: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1504] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 30964 MB memory: -> device: 0, name: Tesla V100-SXM3-32GB-H, pci bus id: 0000:bc:00.0, compute capability: 7.0\n", - "Mon Sep 6 11:45:23 2021[1,1]:2021-09-06 11:45:23.341641: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1504] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 30964 MB memory: -> device: 0, name: Tesla V100-SXM3-32GB-H, pci bus id: 0000:39:00.0, compute capability: 7.0\n", - "Mon Sep 6 11:45:23 2021[1,0]:2021-09-06 11:45:23.346429: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1504] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 30964 MB memory: -> device: 0, name: Tesla V100-SXM3-32GB-H, pci bus id: 0000:36:00.0, compute capability: 7.0\n", - "Mon Sep 6 11:45:23 2021[1,12]:2021-09-06 11:45:23.435210: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1504] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 30964 MB memory: -> device: 0, name: Tesla V100-SXM3-32GB-H, pci bus id: 0000:57:00.0, compute capability: 7.0\n", - "Mon Sep 6 11:45:23 2021[1,9]:2021-09-06 11:45:23.477277: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1504] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 30964 MB memory: -> device: 0, name: Tesla V100-SXM3-32GB-H, pci bus id: 0000:e7:00.0, compute capability: 7.0\n", - "Mon Sep 6 11:45:23 2021[1,5]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/horovod/_keras/callbacks.py:58: UserWarning: Some callbacks may not have access to the averaged metrics, see https://github.com/horovod/horovod/issues/2440\n", - "Mon Sep 6 11:45:23 2021[1,5]: warnings.warn(\n", - "Mon Sep 6 11:45:23 2021[1,5]:2021-09-06 11:45:23.647929: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 11:45:23 2021[1,5]:2021-09-06 11:45:23.647952: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 11:45:23 2021[1,5]:2021-09-06 11:45:23.647993: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1614] Profiler found 1 GPUs\n", - "Mon Sep 6 11:45:23 2021[1,2]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/horovod/_keras/callbacks.py:58: UserWarning: Some callbacks may not have access to the averaged metrics, see https://github.com/horovod/horovod/issues/2440\n", - "Mon Sep 6 11:45:23 2021[1,2]: warnings.warn(\n", - "Mon Sep 6 11:45:23 2021[1,2]:2021-09-06 11:45:23.744042: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 11:45:23 2021[1,2]:2021-09-06 11:45:23.744065: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 11:45:23 2021[1,2]:2021-09-06 11:45:23.744117: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1614] Profiler found 1 GPUs\n", - "Mon Sep 6 11:45:23 2021[1,14]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/horovod/_keras/callbacks.py:58: UserWarning: Some callbacks may not have access to the averaged metrics, see https://github.com/horovod/horovod/issues/2440\n", - "Mon Sep 6 11:45:23 2021[1,14]: warnings.warn(\n", - "Mon Sep 6 11:45:23 2021[1,14]:2021-09-06 11:45:23.759698: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 11:45:23 2021[1,14]:2021-09-06 11:45:23.759719: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 11:45:23 2021[1,14]:2021-09-06 11:45:23.759764: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1614] Profiler found 1 GPUs\n", - "Mon Sep 6 11:45:23 2021[1,13]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/horovod/_keras/callbacks.py:58: UserWarning: Some callbacks may not have access to the averaged metrics, see https://github.com/horovod/horovod/issues/2440\n", - "Mon Sep 6 11:45:23 2021[1,13]: warnings.warn(\n", - "Mon Sep 6 11:45:23 2021[1,13]:2021-09-06 11:45:23.858335: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 11:45:23 2021[1,13]:2021-09-06 11:45:23.858356: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 11:45:23 2021[1,11]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/horovod/_keras/callbacks.py:58: UserWarning: Some callbacks may not have access to the averaged metrics, see https://github.com/horovod/horovod/issues/2440\n", - "Mon Sep 6 11:45:23 2021[1,11]: warnings.warn(\n", - "Mon Sep 6 11:45:23 2021[1,13]:2021-09-06 11:45:23.858399: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1614] Profiler found 1 GPUs\n", - "Mon Sep 6 11:45:23 2021[1,11]:2021-09-06 11:45:23.858710: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 11:45:23 2021[1,11]:2021-09-06 11:45:23.858730: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 11:45:23 2021[1,11]:2021-09-06 11:45:23.858772: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1614] Profiler found 1 GPUs\n", - "Mon Sep 6 11:45:23 2021[1,7]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/horovod/_keras/callbacks.py:58: UserWarning: Some callbacks may not have access to the averaged metrics, see https://github.com/horovod/horovod/issues/2440\n", - "Mon Sep 6 11:45:23 2021[1,7]: warnings.warn(\n", - "Mon Sep 6 11:45:23 2021[1,7]:2021-09-06 11:45:23.940736: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 11:45:23 2021[1,7]:2021-09-06 11:45:23.940759: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 11:45:23 2021[1,7]:2021-09-06 11:45:23.940801: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1614] Profiler found 1 GPUs\n", - "Mon Sep 6 11:45:23 2021[1,10]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/horovod/_keras/callbacks.py:58: UserWarning: Some callbacks may not have access to the averaged metrics, see https://github.com/horovod/horovod/issues/2440\n", - "Mon Sep 6 11:45:23 2021[1,10]: warnings.warn(\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:45:23 2021[1,10]:2021-09-06 11:45:23.941176: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 11:45:23 2021[1,10]:2021-09-06 11:45:23.941195: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 11:45:23 2021[1,10]:2021-09-06 11:45:23.941240: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1614] Profiler found 1 GPUs\n", - "Mon Sep 6 11:45:23 2021[1,5]:2021-09-06 11:45:23.950215: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 11:45:23 2021[1,5]:2021-09-06 11:45:23.952931: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 11:45:23 2021[1,6]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/horovod/_keras/callbacks.py:58: UserWarning: Some callbacks may not have access to the averaged metrics, see https://github.com/horovod/horovod/issues/2440\n", - "Mon Sep 6 11:45:23 2021[1,6]: warnings.warn(\n", - "Mon Sep 6 11:45:23 2021[1,6]:2021-09-06 11:45:23.984430: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 11:45:23 2021[1,6]:2021-09-06 11:45:23.984453: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 11:45:23 2021[1,6]:2021-09-06 11:45:23.984507: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1614] Profiler found 1 GPUs\n", - "Mon Sep 6 11:45:24 2021[1,8]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/horovod/_keras/callbacks.py:58: UserWarning: Some callbacks may not have access to the averaged metrics, see https://github.com/horovod/horovod/issues/2440\n", - "Mon Sep 6 11:45:24 2021[1,8]: warnings.warn(\n", - "Mon Sep 6 11:45:24 2021[1,8]:2021-09-06 11:45:24.013682: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 11:45:24 2021[1,8]:2021-09-06 11:45:24.013706: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 11:45:24 2021[1,8]:2021-09-06 11:45:24.013758: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1614] Profiler found 1 GPUs\n", - "Mon Sep 6 11:45:24 2021[1,3]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/horovod/_keras/callbacks.py:58: UserWarning: Some callbacks may not have access to the averaged metrics, see https://github.com/horovod/horovod/issues/2440\n", - "Mon Sep 6 11:45:24 2021[1,3]: warnings.warn(\n", - "Mon Sep 6 11:45:24 2021[1,3]:2021-09-06 11:45:24.046403: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 11:45:24 2021[1,3]:2021-09-06 11:45:24.046428: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 11:45:24 2021[1,3]:2021-09-06 11:45:24.046476: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1614] Profiler found 1 GPUs\n", - "Mon Sep 6 11:45:24 2021[1,4]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/horovod/_keras/callbacks.py:58: UserWarning: Some callbacks may not have access to the averaged metrics, see https://github.com/horovod/horovod/issues/2440\n", - "Mon Sep 6 11:45:24 2021[1,4]: warnings.warn(\n", - "Mon Sep 6 11:45:24 2021[1,4]:2021-09-06 11:45:24.067336: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 11:45:24 2021[1,4]:2021-09-06 11:45:24.067359: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 11:45:24 2021[1,4]:2021-09-06 11:45:24.067412: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1614] Profiler found 1 GPUs\n", - "Mon Sep 6 11:45:24 2021[1,15]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/horovod/_keras/callbacks.py:58: UserWarning: Some callbacks may not have access to the averaged metrics, see https://github.com/horovod/horovod/issues/2440\n", - "Mon Sep 6 11:45:24 2021[1,15]: warnings.warn(\n", - "Mon Sep 6 11:45:24 2021[1,15]:2021-09-06 11:45:24.118355: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 11:45:24 2021[1,15]:2021-09-06 11:45:24.118379: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 11:45:24 2021[1,15]:2021-09-06 11:45:24.118447: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1614] Profiler found 1 GPUs\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:45:24 2021[1,0]:Model: \"model\"\n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:Layer (type) Output Shape Param # Connected to \n", - "Mon Sep 6 11:45:24 2021[1,0]:==================================================================================================\n", - "Mon Sep 6 11:45:24 2021[1,0]:c0 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:c1 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:c2 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:c3 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:c4 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:c6 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:c7 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:c8 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:c9 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:c10 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:c11 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:c12 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:c13 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:c14 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:c15 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:c17 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:c18 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:c19 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:c20 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:c21 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:c22 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:c23 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:c24 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:45:24 2021[1,1]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/horovod/_keras/callbacks.py:58: UserWarning: Some callbacks may not have access to the averaged metrics, see https://github.com/horovod/horovod/issues/2440\n", - "Mon Sep 6 11:45:24 2021[1,1]: warnings.warn(\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:45:24 2021[1,0]:c25 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:45:24 2021[1,1]:2021-09-06 11:45:24.144886: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:45:24 2021[1,0]:embedding_c0 (Embedding) (None, 1, 8) 63303112 ['c0[0][0]'] \n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:45:24 2021[1,1]:2021-09-06 11:45:24.144909: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 11:45:24 2021[1,1]:2021-09-06 11:45:24.144956: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1614] Profiler found 1 GPUs\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:embedding_c1 (Embedding) (None, 1, 8) 270584 ['c1[0][0]'] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:embedding_c2 (Embedding) (None, 1, 8) 137112 ['c2[0][0]'] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:embedding_c3 (Embedding) (None, 1, 8) 58712 ['c3[0][0]'] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:embedding_c4 (Embedding) (None, 1, 8) 160368 ['c4[0][0]'] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:embedding_c6 (Embedding) (None, 1, 8) 56840 ['c6[0][0]'] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:embedding_c7 (Embedding) (None, 1, 8) 11056 ['c7[0][0]'] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:embedding_c8 (Embedding) (None, 1, 8) 504 ['c8[0][0]'] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:embedding_c9 (Embedding) (None, 1, 8) 44432912 ['c9[0][0]'] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:embedding_c10 (Embedding) (None, 1, 8) 4659752 ['c10[0][0]'] \n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:45:24 2021[1,0]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/horovod/_keras/callbacks.py:58: UserWarning: Some callbacks may not have access to the averaged metrics, see https://github.com/horovod/horovod/issues/2440\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:45:24 2021[1,0]: warnings.warn(\n", - "Mon Sep 6 11:45:24 2021[1,0]:2021-09-06 11:45:24.151145: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:45:24 2021[1,0]:embedding_c11 (Embedding) (None, 1, 8) 1966624 ['c11[0][0]'] \n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:45:24 2021[1,0]:2021-09-06 11:45:24.151167: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:45:24 2021[1,0]:2021-09-06 11:45:24.151215: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1614] Profiler found 1 GPUs\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:45:24 2021[1,0]:embedding_c12 (Embedding) (None, 1, 8) 88 ['c12[0][0]'] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:embedding_c13 (Embedding) (None, 1, 8) 17672 ['c13[0][0]'] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:embedding_c14 (Embedding) (None, 1, 8) 85336 ['c14[0][0]'] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:embedding_c15 (Embedding) (None, 1, 8) 832 ['c15[0][0]'] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:embedding_c17 (Embedding) (None, 1, 8) 7744 ['c17[0][0]'] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:embedding_c18 (Embedding) (None, 1, 8) 120 ['c18[0][0]'] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:embedding_c19 (Embedding) (None, 1, 8) 65327168 ['c19[0][0]'] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:embedding_c20 (Embedding) (None, 1, 8) 21407520 ['c20[0][0]'] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:embedding_c21 (Embedding) (None, 1, 8) 57251624 ['c21[0][0]'] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:embedding_c22 (Embedding) (None, 1, 8) 2420128 ['c22[0][0]'] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:embedding_c23 (Embedding) (None, 1, 8) 96176 ['c23[0][0]'] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:embedding_c24 (Embedding) (None, 1, 8) 776 ['c24[0][0]'] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:embedding_c25 (Embedding) (None, 1, 8) 280 ['c25[0][0]'] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:embeddings_concat (Concatenate (None, 1, 192) 0 ['embedding_c0[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]:) 'embedding_c1[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]: 'embedding_c2[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]: 'embedding_c3[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]: 'embedding_c4[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]: 'embedding_c6[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]: 'embedding_c7[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]: 'embedding_c8[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]: 'embedding_c9[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]: 'embedding_c10[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]: 'embedding_c11[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]: 'embedding_c12[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]: 'embedding_c13[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]: 'embedding_c14[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]: 'embedding_c15[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]: 'embedding_c17[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]: 'embedding_c18[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]: 'embedding_c19[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]: 'embedding_c20[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]: 'embedding_c21[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]: 'embedding_c22[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]: 'embedding_c23[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]: 'embedding_c24[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]: 'embedding_c25[0][0]'] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:c5 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:c16 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:embeddings_flatten (Flatten) (None, 192) 0 ['embeddings_concat[0][0]'] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:one_hot_c5 (CategoryEncoding) (None, 4) 0 ['c5[0][0]'] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:one_hot_c16 (CategoryEncoding) (None, 4) 0 ['c16[0][0]'] \n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:i0 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:i1 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:i2 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:i3 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:i4 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:i5 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:i6 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:i7 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:i8 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:i9 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:i10 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:i11 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:i12 (InputLayer) [(None, 1)] 0 [] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:inputs_concat (Concatenate) (None, 213) 0 ['embeddings_flatten[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]: 'one_hot_c5[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]: 'one_hot_c16[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]: 'i0[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]: 'i1[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]: 'i2[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]: 'i3[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]: 'i4[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]: 'i5[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]: 'i6[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]: 'i7[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]: 'i8[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]: 'i9[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]: 'i10[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]: 'i11[0][0]', \n", - "Mon Sep 6 11:45:24 2021[1,0]: 'i12[0][0]'] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:batch_normalization (BatchNorm (None, 213) 852 ['inputs_concat[0][0]'] \n", - "Mon Sep 6 11:45:24 2021[1,0]:alization) \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:dense (Dense) (None, 64) 13696 ['batch_normalization[0][0]'] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:batch_normalization_1 (BatchNo (None, 64) 256 ['dense[0][0]'] \n", - "Mon Sep 6 11:45:24 2021[1,0]:rmalization) \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:dense_1 (Dense) (None, 64) 4160 ['batch_normalization_1[0][0]'] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:batch_normalization_2 (BatchNo (None, 64) 256 ['dense_1[0][0]'] \n", - "Mon Sep 6 11:45:24 2021[1,0]:rmalization) \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:dense_2 (Dense) (None, 64) 4160 ['batch_normalization_2[0][0]'] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:batch_normalization_3 (BatchNo (None, 64) 256 ['dense_2[0][0]'] \n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:45:24 2021[1,0]:rmalization) \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:dense_3 (Dense) (None, 32) 2080 ['batch_normalization_3[0][0]'] \n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n", - "Mon Sep 6 11:45:24 2021[1,0]:output (Dense) (None, 1) 33 ['dense_3[0][0]'] \n", - "Mon Sep 6 11:45:24 2021[1,0]:==================================================================================================\n", - "Mon Sep 6 11:45:24 2021[1,0]:Total params: 261,698,789\n", - "Mon Sep 6 11:45:24 2021[1,0]:Trainable params: 261,697,979\n", - "Mon Sep 6 11:45:24 2021[1,0]:Non-trainable params: 810\n", - "Mon Sep 6 11:45:24 2021[1,0]:__________________________________________________________________________________________________\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:45:24 2021[1,12]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/horovod/_keras/callbacks.py:58: UserWarning: Some callbacks may not have access to the averaged metrics, see https://github.com/horovod/horovod/issues/2440\n", - "Mon Sep 6 11:45:24 2021[1,12]: warnings.warn(\n", - "Mon Sep 6 11:45:24 2021[1,12]:2021-09-06 11:45:24.193332: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 11:45:24 2021[1,12]:2021-09-06 11:45:24.193353: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 11:45:24 2021[1,12]:2021-09-06 11:45:24.193402: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1614] Profiler found 1 GPUs\n", - "Mon Sep 6 11:45:24 2021[1,14]:2021-09-06 11:45:24.238850: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 11:45:24 2021[1,14]:2021-09-06 11:45:24.241542: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 11:45:24 2021[1,9]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/horovod/_keras/callbacks.py:58: UserWarning: Some callbacks may not have access to the averaged metrics, see https://github.com/horovod/horovod/issues/2440\n", - "Mon Sep 6 11:45:24 2021[1,9]: warnings.warn(\n", - "Mon Sep 6 11:45:24 2021[1,9]:2021-09-06 11:45:24.250555: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 11:45:24 2021[1,9]:2021-09-06 11:45:24.250576: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 11:45:24 2021[1,9]:2021-09-06 11:45:24.250629: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1614] Profiler found 1 GPUs\n", - "Mon Sep 6 11:45:24 2021[1,11]:2021-09-06 11:45:24.257864: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 11:45:24 2021[1,11]:2021-09-06 11:45:24.260597: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 11:45:24 2021[1,2]:2021-09-06 11:45:24.280993: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 11:45:24 2021[1,13]:2021-09-06 11:45:24.287541: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 11:45:24 2021[1,2]:2021-09-06 11:45:24.287675: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 11:45:24 2021[1,13]:2021-09-06 11:45:24.290185: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 11:45:24 2021[1,10]:2021-09-06 11:45:24.509055: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 11:45:24 2021[1,10]:2021-09-06 11:45:24.511856: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 11:45:24 2021[1,7]:2021-09-06 11:45:24.559661: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 11:45:24 2021[1,7]:2021-09-06 11:45:24.562511: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 11:45:24 2021[1,6]:2021-09-06 11:45:24.633007: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 11:45:24 2021[1,6]:2021-09-06 11:45:24.635583: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 11:45:24 2021[1,8]:2021-09-06 11:45:24.856408: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 11:45:24 2021[1,3]:2021-09-06 11:45:24.859705: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 11:45:24 2021[1,8]:2021-09-06 11:45:24.861849: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 11:45:24 2021[1,3]:2021-09-06 11:45:24.864132: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 11:45:24 2021[1,0]:2021-09-06 11:45:24.865753: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 11:45:24 2021[1,0]:2021-09-06 11:45:24.870050: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 11:45:24 2021[1,9]:2021-09-06 11:45:24.920789: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 11:45:24 2021[1,9]:2021-09-06 11:45:24.923768: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 11:45:24 2021[1,12]:2021-09-06 11:45:24.927953: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 11:45:24 2021[1,12]:2021-09-06 11:45:24.930836: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 11:45:24 2021[1,1]:2021-09-06 11:45:24.944178: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 11:45:24 2021[1,1]:2021-09-06 11:45:24.951561: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 11:45:25 2021[1,15]:2021-09-06 11:45:25.019351: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 11:45:25 2021[1,15]:2021-09-06 11:45:25.023739: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 11:45:25 2021[1,4]:2021-09-06 11:45:25.132725: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 11:45:25 2021[1,4]:2021-09-06 11:45:25.139297: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 11:45:25 2021[1,5]:WARNING:tensorflow:AutoGraph could not transform . at 0x7fcebaccea60> and will run it as-is.\n", - "Mon Sep 6 11:45:25 2021[1,5]:Cause: could not parse the source code of . at 0x7fcebaccea60>: no matching AST found\n", - "Mon Sep 6 11:45:25 2021[1,5]:To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n", - "Mon Sep 6 11:45:26 2021[1,5]:WARNING:tensorflow:AutoGraph could not transform . at 0x7fceb96944c0> and will run it as-is.\n", - "Mon Sep 6 11:45:26 2021[1,5]:Cause: could not parse the source code of . at 0x7fceb96944c0>: no matching AST found\n", - "Mon Sep 6 11:45:26 2021[1,5]:To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n", - "Mon Sep 6 11:45:26 2021[1,14]:WARNING:tensorflow:AutoGraph could not transform . at 0x7f7cc813dee0> and will run it as-is.\n", - "Mon Sep 6 11:45:26 2021[1,14]:Cause: could not parse the source code of . at 0x7f7cc813dee0>: no matching AST found\n", - "Mon Sep 6 11:45:26 2021[1,14]:To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n", - "Mon Sep 6 11:45:26 2021[1,2]:WARNING:tensorflow:AutoGraph could not transform . at 0x7f540c075d30> and will run it as-is.\n", - "Mon Sep 6 11:45:26 2021[1,2]:Cause: could not parse the source code of . at 0x7f540c075d30>: no matching AST found\n", - "Mon Sep 6 11:45:26 2021[1,2]:To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n", - "Mon Sep 6 11:45:26 2021[1,11]:WARNING:tensorflow:AutoGraph could not transform . at 0x7f276c530670> and will run it as-is.\n", - "Mon Sep 6 11:45:26 2021[1,11]:Cause: could not parse the source code of . at 0x7f276c530670>: no matching AST found\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:45:26 2021[1,11]:To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n", - "Mon Sep 6 11:45:26 2021[1,13]:WARNING:tensorflow:AutoGraph could not transform . at 0x7f3cc12349d0> and will run it as-is.\n", - "Mon Sep 6 11:45:26 2021[1,13]:Cause: could not parse the source code of . at 0x7f3cc12349d0>: no matching AST found\n", - "Mon Sep 6 11:45:26 2021[1,13]:To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n", - "Mon Sep 6 11:45:26 2021[1,14]:WARNING:tensorflow:AutoGraph could not transform . at 0x7f7bf4174040> and will run it as-is.\n", - "Mon Sep 6 11:45:26 2021[1,14]:Cause: could not parse the source code of . at 0x7f7bf4174040>: no matching AST found\n", - "Mon Sep 6 11:45:26 2021[1,14]:To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n", - "Mon Sep 6 11:45:26 2021[1,2]:WARNING:tensorflow:AutoGraph could not transform . at 0x7f53c4044160> and will run it as-is.\n", - "Mon Sep 6 11:45:26 2021[1,2]:Cause: could not parse the source code of . at 0x7f53c4044160>: no matching AST found\n", - "Mon Sep 6 11:45:26 2021[1,2]:To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n", - "Mon Sep 6 11:45:27 2021[1,11]:WARNING:tensorflow:AutoGraph could not transform . at 0x7f27cc054280> and will run it as-is.\n", - "Mon Sep 6 11:45:27 2021[1,11]:Cause: could not parse the source code of . at 0x7f27cc054280>: no matching AST found\n", - "Mon Sep 6 11:45:27 2021[1,11]:To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n", - "Mon Sep 6 11:45:27 2021[1,13]:WARNING:tensorflow:AutoGraph could not transform . at 0x7f3c04103280> and will run it as-is.\n", - "Mon Sep 6 11:45:27 2021[1,13]:Cause: could not parse the source code of . at 0x7f3c04103280>: no matching AST found\n", - "Mon Sep 6 11:45:27 2021[1,13]:To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n", - "Mon Sep 6 11:45:27 2021[1,7]:WARNING:tensorflow:AutoGraph could not transform . at 0x7fdbaa897550> and will run it as-is.\n", - "Mon Sep 6 11:45:27 2021[1,7]:Cause: could not parse the source code of . at 0x7fdbaa897550>: no matching AST found\n", - "Mon Sep 6 11:45:27 2021[1,7]:To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n", - "Mon Sep 6 11:45:27 2021[1,6]:WARNING:tensorflow:AutoGraph could not transform . at 0x7f24600dbaf0> and will run it as-is.\n", - "Mon Sep 6 11:45:27 2021[1,6]:Cause: could not parse the source code of . at 0x7f24600dbaf0>: no matching AST found\n", - "Mon Sep 6 11:45:27 2021[1,6]:To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n", - "Mon Sep 6 11:45:27 2021[1,10]:WARNING:tensorflow:AutoGraph could not transform . at 0x7f175809ac10> and will run it as-is.\n", - "Mon Sep 6 11:45:27 2021[1,10]:Cause: could not parse the source code of . at 0x7f175809ac10>: no matching AST found\n", - "Mon Sep 6 11:45:27 2021[1,10]:To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n", - "Mon Sep 6 11:45:27 2021[1,6]:WARNING:tensorflow:AutoGraph could not transform . at 0x7f2460074280> and will run it as-is.\n", - "Mon Sep 6 11:45:27 2021[1,6]:Cause: could not parse the source code of . at 0x7f2460074280>: no matching AST found\n", - "Mon Sep 6 11:45:27 2021[1,6]:To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n", - "Mon Sep 6 11:45:27 2021[1,7]:WARNING:tensorflow:AutoGraph could not transform . at 0x7fdc342ec280> and will run it as-is.\n", - "Mon Sep 6 11:45:27 2021[1,7]:Cause: could not parse the source code of . at 0x7fdc342ec280>: no matching AST found\n", - "Mon Sep 6 11:45:27 2021[1,7]:To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n", - "Mon Sep 6 11:45:27 2021[1,10]:WARNING:tensorflow:AutoGraph could not transform . at 0x7f17a81bf5e0> and will run it as-is.\n", - "Mon Sep 6 11:45:27 2021[1,10]:Cause: could not parse the source code of . at 0x7f17a81bf5e0>: no matching AST found\n", - "Mon Sep 6 11:45:27 2021[1,10]:To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n", - "Mon Sep 6 11:45:28 2021[1,0]:WARNING:tensorflow:AutoGraph could not transform . at 0x7f1ae4675ee0> and will run it as-is.\n", - "Mon Sep 6 11:45:28 2021[1,0]:Cause: could not parse the source code of . at 0x7f1ae4675ee0>: no matching AST found\n", - "Mon Sep 6 11:45:28 2021[1,0]:To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n", - "Mon Sep 6 11:45:28 2021[1,3]:WARNING:tensorflow:AutoGraph could not transform . at 0x7f32a406d550> and will run it as-is.\n", - "Mon Sep 6 11:45:28 2021[1,3]:Cause: could not parse the source code of . at 0x7f32a406d550>: no matching AST found\n", - "Mon Sep 6 11:45:28 2021[1,3]:To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n", - "Mon Sep 6 11:45:28 2021[1,1]:WARNING:tensorflow:AutoGraph could not transform . at 0x7fd8c81b6b80> and will run it as-is.\n", - "Mon Sep 6 11:45:28 2021[1,1]:Cause: could not parse the source code of . at 0x7fd8c81b6b80>: no matching AST found\n", - "Mon Sep 6 11:45:28 2021[1,1]:To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n", - "Mon Sep 6 11:45:28 2021[1,8]:WARNING:tensorflow:AutoGraph could not transform . at 0x7fd84e254dc0> and will run it as-is.\n", - "Mon Sep 6 11:45:28 2021[1,8]:Cause: could not parse the source code of . at 0x7fd84e254dc0>: no matching AST found\n", - "Mon Sep 6 11:45:28 2021[1,8]:To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n", - "Mon Sep 6 11:45:28 2021[1,15]:WARNING:tensorflow:AutoGraph could not transform . at 0x7fe3a71ee1f0> and will run it as-is.\n", - "Mon Sep 6 11:45:28 2021[1,15]:Cause: could not parse the source code of . at 0x7fe3a71ee1f0>: no matching AST found\n", - "Mon Sep 6 11:45:28 2021[1,15]:To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n", - "Mon Sep 6 11:45:28 2021[1,9]:WARNING:tensorflow:AutoGraph could not transform . at 0x7fb74018b550> and will run it as-is.\n", - "Mon Sep 6 11:45:28 2021[1,9]:Cause: could not parse the source code of . at 0x7fb74018b550>: no matching AST found\n", - "Mon Sep 6 11:45:28 2021[1,9]:To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n", - "Mon Sep 6 11:45:28 2021[1,0]:WARNING:tensorflow:AutoGraph could not transform . at 0x7f1bd00ae280> and will run it as-is.\n", - "Mon Sep 6 11:45:28 2021[1,0]:Cause: could not parse the source code of . at 0x7f1bd00ae280>: no matching AST found\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:45:28 2021[1,0]:To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n", - "Mon Sep 6 11:45:28 2021[1,12]:WARNING:tensorflow:AutoGraph could not transform . at 0x7fce00127ca0> and will run it as-is.\n", - "Mon Sep 6 11:45:28 2021[1,12]:Cause: could not parse the source code of . at 0x7fce00127ca0>: no matching AST found\n", - "Mon Sep 6 11:45:28 2021[1,12]:To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n", - "Mon Sep 6 11:45:28 2021[1,8]:WARNING:tensorflow:AutoGraph could not transform . at 0x7fd7b01be430> and will run it as-is.\n", - "Mon Sep 6 11:45:28 2021[1,8]:Cause: could not parse the source code of . at 0x7fd7b01be430>: no matching AST found\n", - "Mon Sep 6 11:45:28 2021[1,8]:To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n", - "Mon Sep 6 11:45:28 2021[1,4]:WARNING:tensorflow:AutoGraph could not transform . at 0x7f2a840e49d0> and will run it as-is.\n", - "Mon Sep 6 11:45:28 2021[1,4]:Cause: could not parse the source code of . at 0x7f2a840e49d0>: no matching AST found\n", - "Mon Sep 6 11:45:28 2021[1,4]:To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n", - "Mon Sep 6 11:45:28 2021[1,3]:WARNING:tensorflow:AutoGraph could not transform . at 0x7f3254018280> and will run it as-is.\n", - "Mon Sep 6 11:45:28 2021[1,3]:Cause: could not parse the source code of . at 0x7f3254018280>: no matching AST found\n", - "Mon Sep 6 11:45:28 2021[1,3]:To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n", - "Mon Sep 6 11:45:28 2021[1,1]:WARNING:tensorflow:AutoGraph could not transform . at 0x7fd91404c280> and will run it as-is.\n", - "Mon Sep 6 11:45:28 2021[1,1]:Cause: could not parse the source code of . at 0x7fd91404c280>: no matching AST found\n", - "Mon Sep 6 11:45:28 2021[1,1]:To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n", - "Mon Sep 6 11:45:28 2021[1,12]:WARNING:tensorflow:AutoGraph could not transform . at 0x7fcdac026280> and will run it as-is.\n", - "Mon Sep 6 11:45:28 2021[1,12]:Cause: could not parse the source code of . at 0x7fcdac026280>: no matching AST found\n", - "Mon Sep 6 11:45:28 2021[1,12]:To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n", - "Mon Sep 6 11:45:28 2021[1,15]:WARNING:tensorflow:AutoGraph could not transform . at 0x7fe2d40f7280> and will run it as-is.\n", - "Mon Sep 6 11:45:28 2021[1,15]:Cause: could not parse the source code of . at 0x7fe2d40f7280>: no matching AST found\n", - "Mon Sep 6 11:45:28 2021[1,15]:To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n", - "Mon Sep 6 11:45:28 2021[1,4]:WARNING:tensorflow:AutoGraph could not transform . at 0x7f2a64056160> and will run it as-is.\n", - "Mon Sep 6 11:45:28 2021[1,4]:Cause: could not parse the source code of . at 0x7f2a64056160>: no matching AST found\n", - "Mon Sep 6 11:45:28 2021[1,4]:To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n", - "Mon Sep 6 11:45:28 2021[1,9]:WARNING:tensorflow:AutoGraph could not transform . at 0x7fb740149040> and will run it as-is.\n", - "Mon Sep 6 11:45:28 2021[1,9]:Cause: could not parse the source code of . at 0x7fb740149040>: no matching AST found\n", - "Mon Sep 6 11:45:28 2021[1,9]:To silence this warning, decorate the function with @tf.autograph.experimental.do_not_convert\n", - "Mon Sep 6 11:45:36 2021[1,14]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/petastorm/arrow_reader_worker.py:53: FutureWarning: Calling .data on ChunkedArray is provided for compatibility after Column was removed, simply drop this attribute\n", - "Mon Sep 6 11:45:36 2021[1,14]: column_as_pandas = column.data.chunks[0].to_pandas()\n", - "Mon Sep 6 11:45:36 2021[1,2]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/petastorm/arrow_reader_worker.py:53: FutureWarning: Calling .data on ChunkedArray is provided for compatibility after Column was removed, simply drop this attribute\n", - "Mon Sep 6 11:45:36 2021[1,2]: column_as_pandas = column.data.chunks[0].to_pandas()\n", - "Mon Sep 6 11:45:36 2021[1,11]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/petastorm/arrow_reader_worker.py:53: FutureWarning: Calling .data on ChunkedArray is provided for compatibility after Column was removed, simply drop this attribute\n", - "Mon Sep 6 11:45:36 2021[1,11]: column_as_pandas = column.data.chunks[0].to_pandas()\n", - "Mon Sep 6 11:45:36 2021[1,5]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/petastorm/arrow_reader_worker.py:53: FutureWarning: Calling .data on ChunkedArray is provided for compatibility after Column was removed, simply drop this attribute\n", - "Mon Sep 6 11:45:36 2021[1,5]: column_as_pandas = column.data.chunks[0].to_pandas()\n", - "Mon Sep 6 11:45:36 2021[1,6]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/petastorm/arrow_reader_worker.py:53: FutureWarning: Calling .data on ChunkedArray is provided for compatibility after Column was removed, simply drop this attribute\n", - "Mon Sep 6 11:45:36 2021[1,6]: column_as_pandas = column.data.chunks[0].to_pandas()\n", - "Mon Sep 6 11:45:36 2021[1,10]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/petastorm/arrow_reader_worker.py:53: FutureWarning: Calling .data on ChunkedArray is provided for compatibility after Column was removed, simply drop this attribute\n", - "Mon Sep 6 11:45:36 2021[1,10]: column_as_pandas = column.data.chunks[0].to_pandas()\n", - "Mon Sep 6 11:45:36 2021[1,7]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/petastorm/arrow_reader_worker.py:53: FutureWarning: Calling .data on ChunkedArray is provided for compatibility after Column was removed, simply drop this attribute\n", - "Mon Sep 6 11:45:36 2021[1,7]: column_as_pandas = column.data.chunks[0].to_pandas()\n", - "Mon Sep 6 11:45:36 2021[1,13]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/petastorm/arrow_reader_worker.py:53: FutureWarning: Calling .data on ChunkedArray is provided for compatibility after Column was removed, simply drop this attribute\n", - "Mon Sep 6 11:45:36 2021[1,13]: column_as_pandas = column.data.chunks[0].to_pandas()\n", - "Mon Sep 6 11:45:37 2021[1,8]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/petastorm/arrow_reader_worker.py:53: FutureWarning: Calling .data on ChunkedArray is provided for compatibility after Column was removed, simply drop this attribute\n", - "Mon Sep 6 11:45:37 2021[1,8]: column_as_pandas = column.data.chunks[0].to_pandas()\n", - "Mon Sep 6 11:45:37 2021[1,4]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/petastorm/arrow_reader_worker.py:53: FutureWarning: Calling .data on ChunkedArray is provided for compatibility after Column was removed, simply drop this attribute\n", - "Mon Sep 6 11:45:37 2021[1,4]: column_as_pandas = column.data.chunks[0].to_pandas()\n", - "Mon Sep 6 11:45:37 2021[1,0]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/petastorm/arrow_reader_worker.py:53: FutureWarning: Calling .data on ChunkedArray is provided for compatibility after Column was removed, simply drop this attribute\n", - "Mon Sep 6 11:45:37 2021[1,0]: column_as_pandas = column.data.chunks[0].to_pandas()\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:45:37 2021[1,3]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/petastorm/arrow_reader_worker.py:53: FutureWarning: Calling .data on ChunkedArray is provided for compatibility after Column was removed, simply drop this attribute\n", - "Mon Sep 6 11:45:37 2021[1,3]: column_as_pandas = column.data.chunks[0].to_pandas()\n", - "Mon Sep 6 11:45:37 2021[1,1]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/petastorm/arrow_reader_worker.py:53: FutureWarning: Calling .data on ChunkedArray is provided for compatibility after Column was removed, simply drop this attribute\n", - "Mon Sep 6 11:45:37 2021[1,1]: column_as_pandas = column.data.chunks[0].to_pandas()\n", - "Mon Sep 6 11:45:37 2021[1,12]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/petastorm/arrow_reader_worker.py:53: FutureWarning: Calling .data on ChunkedArray is provided for compatibility after Column was removed, simply drop this attribute\n", - "Mon Sep 6 11:45:37 2021[1,12]: column_as_pandas = column.data.chunks[0].to_pandas()\n", - "Mon Sep 6 11:45:37 2021[1,15]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/petastorm/arrow_reader_worker.py:53: FutureWarning: Calling .data on ChunkedArray is provided for compatibility after Column was removed, simply drop this attribute\n", - "Mon Sep 6 11:45:37 2021[1,15]: column_as_pandas = column.data.chunks[0].to_pandas()\n", - "Mon Sep 6 11:45:37 2021[1,9]:/home/ngc-auth-ldap-allxu/miniconda3/lib/python3.8/site-packages/petastorm/arrow_reader_worker.py:53: FutureWarning: Calling .data on ChunkedArray is provided for compatibility after Column was removed, simply drop this attribute\n", - "Mon Sep 6 11:45:37 2021[1,9]: column_as_pandas = column.data.chunks[0].to_pandas()\n", - "Mon Sep 6 11:45:46 2021[1,14]:2021-09-06 11:45:46.163581: I tensorflow/core/kernels/data/shuffle_dataset_op.cc:338] Filling up shuffle buffer (this may take a while): 633106 of 655350\n", - "Mon Sep 6 11:45:46 2021[1,2]:2021-09-06 11:45:46.173808: I tensorflow/core/kernels/data/shuffle_dataset_op.cc:338] Filling up shuffle buffer (this may take a while): 625961 of 655350\n", - "Mon Sep 6 11:45:46 2021[1,11]:2021-09-06 11:45:46.470637: I tensorflow/core/kernels/data/shuffle_dataset_op.cc:338] Filling up shuffle buffer (this may take a while): 575620 of 655350\n", - "Mon Sep 6 11:45:46 2021[1,5]:2021-09-06 11:45:46.483315: I tensorflow/core/kernels/data/shuffle_dataset_op.cc:338] Filling up shuffle buffer (this may take a while): 602125 of 655350\n", - "Mon Sep 6 11:45:46 2021[1,14]:2021-09-06 11:45:46.515473: I tensorflow/core/kernels/data/shuffle_dataset_op.cc:363] Shuffle buffer filled.\n", - "Mon Sep 6 11:45:46 2021[1,6]:2021-09-06 11:45:46.620458: I tensorflow/core/kernels/data/shuffle_dataset_op.cc:338] Filling up shuffle buffer (this may take a while): 612890 of 655350\n", - "Mon Sep 6 11:45:46 2021[1,2]:2021-09-06 11:45:46.655265: I tensorflow/core/kernels/data/shuffle_dataset_op.cc:363] Shuffle buffer filled.\n", - "Mon Sep 6 11:45:46 2021[1,10]:2021-09-06 11:45:46.681795: I tensorflow/core/kernels/data/shuffle_dataset_op.cc:338] Filling up shuffle buffer (this may take a while): 601332 of 655350\n", - "Mon Sep 6 11:45:46 2021[1,7]:2021-09-06 11:45:46.708487: I tensorflow/core/kernels/data/shuffle_dataset_op.cc:338] Filling up shuffle buffer (this may take a while): 572490 of 655350\n", - "Mon Sep 6 11:45:46 2021[1,13]:2021-09-06 11:45:46.848852: I tensorflow/core/kernels/data/shuffle_dataset_op.cc:338] Filling up shuffle buffer (this may take a while): 584226 of 655350\n", - "Mon Sep 6 11:45:47 2021[1,8]:2021-09-06 11:45:47.028900: I tensorflow/core/kernels/data/shuffle_dataset_op.cc:338] Filling up shuffle buffer (this may take a while): 631682 of 655350\n", - "Mon Sep 6 11:45:47 2021[1,4]:2021-09-06 11:45:47.123354: I tensorflow/core/kernels/data/shuffle_dataset_op.cc:338] Filling up shuffle buffer (this may take a while): 622824 of 655350\n", - "Mon Sep 6 11:45:47 2021[1,0]:2021-09-06 11:45:47.160067: I tensorflow/core/kernels/data/shuffle_dataset_op.cc:338] Filling up shuffle buffer (this may take a while): 624462 of 655350\n", - "Mon Sep 6 11:45:47 2021[1,3]:2021-09-06 11:45:47.306279: I tensorflow/core/kernels/data/shuffle_dataset_op.cc:338] Filling up shuffle buffer (this may take a while): 559646 of 655350\n", - "Mon Sep 6 11:45:47 2021[1,6]:2021-09-06 11:45:47.317935: I tensorflow/core/kernels/data/shuffle_dataset_op.cc:363] Shuffle buffer filled.\n", - "Mon Sep 6 11:45:47 2021[1,1]:2021-09-06 11:45:47.328390: I tensorflow/core/kernels/data/shuffle_dataset_op.cc:338] Filling up shuffle buffer (this may take a while): 602957 of 655350\n", - "Mon Sep 6 11:45:47 2021[1,12]:2021-09-06 11:45:47.401105: I tensorflow/core/kernels/data/shuffle_dataset_op.cc:338] Filling up shuffle buffer (this may take a while): 609749 of 655350\n", - "Mon Sep 6 11:45:47 2021[1,8]:2021-09-06 11:45:47.406648: I tensorflow/core/kernels/data/shuffle_dataset_op.cc:363] Shuffle buffer filled.\n", - "Mon Sep 6 11:45:47 2021[1,5]:2021-09-06 11:45:47.410203: I tensorflow/core/kernels/data/shuffle_dataset_op.cc:363] Shuffle buffer filled.\n", - "Mon Sep 6 11:45:47 2021[1,15]:2021-09-06 11:45:47.490260: I tensorflow/core/kernels/data/shuffle_dataset_op.cc:338] Filling up shuffle buffer (this may take a while): 606865 of 655350\n", - "Mon Sep 6 11:45:47 2021[1,10]:2021-09-06 11:45:47.576105: I tensorflow/core/kernels/data/shuffle_dataset_op.cc:363] Shuffle buffer filled.\n", - "Mon Sep 6 11:45:47 2021[1,9]:2021-09-06 11:45:47.631347: I tensorflow/core/kernels/data/shuffle_dataset_op.cc:338] Filling up shuffle buffer (this may take a while): 610214 of 655350\n", - "Mon Sep 6 11:45:47 2021[1,4]:2021-09-06 11:45:47.638664: I tensorflow/core/kernels/data/shuffle_dataset_op.cc:363] Shuffle buffer filled.\n", - "Mon Sep 6 11:45:47 2021[1,0]:2021-09-06 11:45:47.648328: I tensorflow/core/kernels/data/shuffle_dataset_op.cc:363] Shuffle buffer filled.\n", - "Mon Sep 6 11:45:47 2021[1,11]:2021-09-06 11:45:47.874307: I tensorflow/core/kernels/data/shuffle_dataset_op.cc:363] Shuffle buffer filled.\n", - "Mon Sep 6 11:45:48 2021[1,12]:2021-09-06 11:45:48.136479: I tensorflow/core/kernels/data/shuffle_dataset_op.cc:363] Shuffle buffer filled.\n", - "Mon Sep 6 11:45:48 2021[1,7]:2021-09-06 11:45:48.178890: I tensorflow/core/kernels/data/shuffle_dataset_op.cc:363] Shuffle buffer filled.\n", - "Mon Sep 6 11:45:48 2021[1,1]:2021-09-06 11:45:48.243785: I tensorflow/core/kernels/data/shuffle_dataset_op.cc:363] Shuffle buffer filled.\n", - "Mon Sep 6 11:45:48 2021[1,15]:2021-09-06 11:45:48.283844: I tensorflow/core/kernels/data/shuffle_dataset_op.cc:363] Shuffle buffer filled.\n", - "Mon Sep 6 11:45:48 2021[1,13]:2021-09-06 11:45:48.332091: I tensorflow/core/kernels/data/shuffle_dataset_op.cc:363] Shuffle buffer filled.\n", - "Mon Sep 6 11:45:48 2021[1,9]:2021-09-06 11:45:48.383569: I tensorflow/core/kernels/data/shuffle_dataset_op.cc:363] Shuffle buffer filled.\n", - "Mon Sep 6 11:45:49 2021[1,3]:2021-09-06 11:45:49.210125: I tensorflow/core/kernels/data/shuffle_dataset_op.cc:363] Shuffle buffer filled.\n", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:45:53 2021[1,0]:dgx2h0194:41886:42730 [0] NCCL INFO Bootstrap : Using enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 11:45:53 2021[1,0]:dgx2h0194:41886:42730 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation\n", - "Mon Sep 6 11:45:53 2021[1,0]:dgx2h0194:41886:42730 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [4]mlx5_4:1/RoCE [5]mlx5_5:1/RoCE [6]mlx5_6:1/RoCE [7]mlx5_7:1/RoCE [8]mlx5_8:1/RoCE [9]mlx5_9:1/RoCE ; OOB enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 11:45:53 2021[1,0]:dgx2h0194:41886:42730 [0] NCCL INFO Using network IB\n", - "Mon Sep 6 11:45:53 2021[1,0]:NCCL version 2.10.3+cuda11.0\n", - "Mon Sep 6 11:45:53 2021[1,6]:dgx2h0194:41892:42733 [0] NCCL INFO Bootstrap : Using enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 11:45:53 2021[1,11]:dgx2h0194:41897:42858 [0] NCCL INFO Bootstrap : Using enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 11:45:53 2021[1,6]:dgx2h0194:41892:42733 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation\n", - "Mon Sep 6 11:45:53 2021[1,7]:dgx2h0194:41893:42743 [0] NCCL INFO Bootstrap : Using enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 11:45:53 2021[1,5]:dgx2h0194:41891:42723 [0] NCCL INFO Bootstrap : Using enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 11:45:53 2021[1,10]:dgx2h0194:41896:42859 [0] NCCL INFO Bootstrap : Using enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 11:45:53 2021[1,12]:dgx2h0194:41898:42413 [0] NCCL INFO Bootstrap : Using enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 11:45:53 2021[1,1]:dgx2h0194:41887:42486 [0] NCCL INFO Bootstrap : Using enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 11:45:53 2021[1,4]:dgx2h0194:41890:42739 [0] NCCL INFO Bootstrap : Using enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 11:45:53 2021[1,9]:dgx2h0194:41895:42860 [0] NCCL INFO Bootstrap : Using enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 11:45:53 2021[1,14]:dgx2h0194:41900:42734 [0] NCCL INFO Bootstrap : Using enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 11:45:53 2021[1,8]:dgx2h0194:41894:42489 [0] NCCL INFO Bootstrap : Using enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 11:45:53 2021[1,13]:dgx2h0194:41899:42721 [0] NCCL INFO Bootstrap : Using enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 11:45:53 2021[1,2]:dgx2h0194:41888:42361 [0] NCCL INFO Bootstrap : Using enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 11:45:53 2021[1,15]:dgx2h0194:41901:42857 [0] NCCL INFO Bootstrap : Using enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 11:45:53 2021[1,11]:dgx2h0194:41897:42858 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation\n", - "Mon Sep 6 11:45:53 2021[1,1]:dgx2h0194:41887:42486 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation\n", - "Mon Sep 6 11:45:53 2021[1,5]:dgx2h0194:41891:42723 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation\n", - "Mon Sep 6 11:45:53 2021[1,12]:dgx2h0194:41898:42413 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation\n", - "Mon Sep 6 11:45:53 2021[1,13]:dgx2h0194:41899:42721 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation\n", - "Mon Sep 6 11:45:53 2021[1,7]:dgx2h0194:41893:42743 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation\n", - "Mon Sep 6 11:45:53 2021[1,10]:dgx2h0194:41896:42859 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation\n", - "Mon Sep 6 11:45:53 2021[1,8]:dgx2h0194:41894:42489 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation\n", - "Mon Sep 6 11:45:53 2021[1,9]:dgx2h0194:41895:42860 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation\n", - "Mon Sep 6 11:45:53 2021[1,14]:dgx2h0194:41900:42734 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation\n", - "Mon Sep 6 11:45:53 2021[1,2]:dgx2h0194:41888:42361 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation\n", - "Mon Sep 6 11:45:53 2021[1,4]:dgx2h0194:41890:42739 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation\n", - "Mon Sep 6 11:45:53 2021[1,15]:dgx2h0194:41901:42857 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation\n", - "Mon Sep 6 11:45:53 2021[1,6]:dgx2h0194:41892:42733 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [4]mlx5_4:1/RoCE [5]mlx5_5:1/RoCE [6]mlx5_6:1/RoCE [7]mlx5_7:1/RoCE [8]mlx5_8:1/RoCE [9]mlx5_9:1/RoCE ; OOB enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 11:45:53 2021[1,6]:dgx2h0194:41892:42733 [0] NCCL INFO Using network IB\n", - "Mon Sep 6 11:45:53 2021[1,11]:dgx2h0194:41897:42858 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [4]mlx5_4:1/RoCE [5]mlx5_5:1/RoCE [6]mlx5_6:1/RoCE [7]mlx5_7:1/RoCE [8]mlx5_8:1/RoCE [9]mlx5_9:1/RoCE ; OOB enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 11:45:53 2021[1,11]:dgx2h0194:41897:42858 [0] NCCL INFO Using network IB\n", - "Mon Sep 6 11:45:53 2021[1,1]:dgx2h0194:41887:42486 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [4]mlx5_4:1/RoCE [5]mlx5_5:1/RoCE [6]mlx5_6:1/RoCE [7]mlx5_7:1/RoCE [8]mlx5_8:1/RoCE [9]mlx5_9:1/RoCE ; OOB enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 11:45:53 2021[1,1]:dgx2h0194:41887:42486 [0] NCCL INFO Using network IB\n", - "Mon Sep 6 11:45:53 2021[1,5]:dgx2h0194:41891:42723 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [4]mlx5_4:1/RoCE [5]mlx5_5:1/RoCE [6]mlx5_6:1/RoCE [7]mlx5_7:1/RoCE [8]mlx5_8:1/RoCE [9]mlx5_9:1/RoCE ; OOB enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 11:45:53 2021[1,5]:dgx2h0194:41891:42723 [0] NCCL INFO Using network IB\n", - "Mon Sep 6 11:45:53 2021[1,12]:dgx2h0194:41898:42413 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [4]mlx5_4:1/RoCE [5]mlx5_5:1/RoCE [6]mlx5_6:1/RoCE [7]mlx5_7:1/RoCE [8]mlx5_8:1/RoCE [9]mlx5_9:1/RoCE ; OOB enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 11:45:53 2021[1,12]:dgx2h0194:41898:42413 [0] NCCL INFO Using network IB\n", - "Mon Sep 6 11:45:53 2021[1,4]:dgx2h0194:41890:42739 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [4]mlx5_4:1/RoCE [5]mlx5_5:1/RoCE [6]mlx5_6:1/RoCE [7]mlx5_7:1/RoCE [8]mlx5_8:1/RoCE [9]mlx5_9:1/RoCE ; OOB enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 11:45:53 2021[1,4]:dgx2h0194:41890:42739 [0] NCCL INFO Using network IB\n", - "Mon Sep 6 11:45:53 2021[1,8]:dgx2h0194:41894:42489 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [4]mlx5_4:1/RoCE [5]mlx5_5:1/RoCE [6]mlx5_6:1/RoCE [7]mlx5_7:1/RoCE [8]mlx5_8:1/RoCE [9]mlx5_9:1/RoCE ; OOB enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 11:45:53 2021[1,8]:dgx2h0194:41894:42489 [0] NCCL INFO Using network IB\n", - "Mon Sep 6 11:45:53 2021[1,13]:dgx2h0194:41899:42721 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [4]mlx5_4:1/RoCE [5]mlx5_5:1/RoCE [6]mlx5_6:1/RoCE [7]mlx5_7:1/RoCE [8]mlx5_8:1/RoCE [9]mlx5_9:1/RoCE ; OOB enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 11:45:53 2021[1,13]:dgx2h0194:41899:42721 [0] NCCL INFO Using network IB\n", - "Mon Sep 6 11:45:53 2021[1,9]:dgx2h0194:41895:42860 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [4]mlx5_4:1/RoCE [5]mlx5_5:1/RoCE [6]mlx5_6:1/RoCE [7]mlx5_7:1/RoCE [8]mlx5_8:1/RoCE [9]mlx5_9:1/RoCE ; OOB enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 11:45:53 2021[1,9]:dgx2h0194:41895:42860 [0] NCCL INFO Using network IB\n", - "Mon Sep 6 11:45:53 2021[1,14]:dgx2h0194:41900:42734 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [4]mlx5_4:1/RoCE [5]mlx5_5:1/RoCE [6]mlx5_6:1/RoCE [7]mlx5_7:1/RoCE [8]mlx5_8:1/RoCE [9]mlx5_9:1/RoCE ; OOB enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 11:45:53 2021[1,14]:dgx2h0194:41900:42734 [0] NCCL INFO Using network IB\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:45:53 2021[1,10]:dgx2h0194:41896:42859 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [4]mlx5_4:1/RoCE [5]mlx5_5:1/RoCE [6]mlx5_6:1/RoCE [7]mlx5_7:1/RoCE [8]mlx5_8:1/RoCE [9]mlx5_9:1/RoCE ; OOB enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 11:45:53 2021[1,10]:dgx2h0194:41896:42859 [0] NCCL INFO Using network IB\n", - "Mon Sep 6 11:45:53 2021[1,15]:dgx2h0194:41901:42857 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [4]mlx5_4:1/RoCE [5]mlx5_5:1/RoCE [6]mlx5_6:1/RoCE [7]mlx5_7:1/RoCE [8]mlx5_8:1/RoCE [9]mlx5_9:1/RoCE ; OOB enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 11:45:53 2021[1,15]:dgx2h0194:41901:42857 [0] NCCL INFO Using network IB\n", - "Mon Sep 6 11:45:53 2021[1,2]:dgx2h0194:41888:42361 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [4]mlx5_4:1/RoCE [5]mlx5_5:1/RoCE [6]mlx5_6:1/RoCE [7]mlx5_7:1/RoCE [8]mlx5_8:1/RoCE [9]mlx5_9:1/RoCE ; OOB enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 11:45:53 2021[1,2]:dgx2h0194:41888:42361 [0] NCCL INFO Using network IB\n", - "Mon Sep 6 11:45:53 2021[1,7]:dgx2h0194:41893:42743 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [4]mlx5_4:1/RoCE [5]mlx5_5:1/RoCE [6]mlx5_6:1/RoCE [7]mlx5_7:1/RoCE [8]mlx5_8:1/RoCE [9]mlx5_9:1/RoCE ; OOB enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 11:45:53 2021[1,7]:dgx2h0194:41893:42743 [0] NCCL INFO Using network IB\n", - "Mon Sep 6 11:45:53 2021[1,3]:dgx2h0194:41889:42722 [0] NCCL INFO Bootstrap : Using enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 11:45:53 2021[1,3]:dgx2h0194:41889:42722 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation\n", - "Mon Sep 6 11:45:53 2021[1,3]:dgx2h0194:41889:42722 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [4]mlx5_4:1/RoCE [5]mlx5_5:1/RoCE [6]mlx5_6:1/RoCE [7]mlx5_7:1/RoCE [8]mlx5_8:1/RoCE [9]mlx5_9:1/RoCE ; OOB enp53s0:10.148.30.59<0>\n", - "Mon Sep 6 11:45:53 2021[1,3]:dgx2h0194:41889:42722 [0] NCCL INFO Using network IB\n", - "Mon Sep 6 11:45:53 2021[1,11]:dgx2h0194:41897:42858 [0] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.\n", - "Mon Sep 6 11:45:53 2021[1,10]:dgx2h0194:41896:42859 [0] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.\n", - "Mon Sep 6 11:45:53 2021[1,13]:dgx2h0194:41899:42721 [0] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.\n", - "Mon Sep 6 11:45:53 2021[1,14]:dgx2h0194:41900:42734 [0] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.\n", - "Mon Sep 6 11:45:53 2021[1,0]:dgx2h0194:41886:42730 [0] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.\n", - "Mon Sep 6 11:45:53 2021[1,1]:dgx2h0194:41887:42486 [0] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.\n", - "Mon Sep 6 11:45:53 2021[1,12]:dgx2h0194:41898:42413 [0] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.\n", - "Mon Sep 6 11:45:53 2021[1,8]:dgx2h0194:41894:42489 [0] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.\n", - "Mon Sep 6 11:45:53 2021[1,15]:dgx2h0194:41901:42857 [0] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.\n", - "Mon Sep 6 11:45:53 2021[1,6]:dgx2h0194:41892:42733 [0] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.\n", - "Mon Sep 6 11:45:53 2021[1,5]:dgx2h0194:41891:42723 [0] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.\n", - "Mon Sep 6 11:45:53 2021[1,7]:dgx2h0194:41893:42743 [0] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.\n", - "Mon Sep 6 11:45:53 2021[1,4]:dgx2h0194:41890:42739 [0] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.\n", - "Mon Sep 6 11:45:53 2021[1,9]:dgx2h0194:41895:42860 [0] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.\n", - "Mon Sep 6 11:45:53 2021[1,2]:dgx2h0194:41888:42361 [0] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.\n", - "Mon Sep 6 11:45:53 2021[1,3]:dgx2h0194:41889:42722 [0] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.\n", - "Mon Sep 6 11:45:58 2021[1,13]:dgx2h0194:41899:42721 [0] NCCL INFO Trees [0] 1/-1/-1->13->0 [1] 1/-1/-1->13->0 [2] 1/-1/-1->13->0 [3] 1/-1/-1->13->0 [4] 1/-1/-1->13->0 [5] 1/-1/-1->13->0 [6] 1/-1/-1->13->0 [7] 1/-1/-1->13->0 [8] 1/-1/-1->13->0 [9] 1/-1/-1->13->0 [10] 1/-1/-1->13->0 [11] 1/-1/-1->13->0\n", - "Mon Sep 6 11:45:58 2021[1,13]:dgx2h0194:41899:42721 [0] NCCL INFO Setting affinity for GPU 0 to ff,ffff0000,00ffffff\n", - "Mon Sep 6 11:45:58 2021[1,0]:dgx2h0194:41886:42730 [0] NCCL INFO Channel 00/12 : 0 13 1 14 5 12 7 11 2 9 3 10 4 8 6 15\n", - "Mon Sep 6 11:45:58 2021[1,0]:dgx2h0194:41886:42730 [0] NCCL INFO Channel 01/12 : 0 13 1 14 5 12 7 11 2 9 3 10 4 8 6 15\n", - "Mon Sep 6 11:45:58 2021[1,0]:dgx2h0194:41886:42730 [0] NCCL INFO Channel 02/12 : 0 13 1 14 5 12 7 11 2 9 3 10 4 8 6 15\n", - "Mon Sep 6 11:45:58 2021[1,0]:dgx2h0194:41886:42730 [0] NCCL INFO Channel 03/12 : 0 13 1 14 5 12 7 11 2 9 3 10 4 8 6 15\n", - "Mon Sep 6 11:45:58 2021[1,0]:dgx2h0194:41886:42730 [0] NCCL INFO Channel 04/12 : 0 13 1 14 5 12 7 11 2 9 3 10 4 8 6 15\n", - "Mon Sep 6 11:45:58 2021[1,0]:dgx2h0194:41886:42730 [0] NCCL INFO Channel 05/12 : 0 13 1 14 5 12 7 11 2 9 3 10 4 8 6 15\n", - "Mon Sep 6 11:45:58 2021[1,0]:dgx2h0194:41886:42730 [0] NCCL INFO Channel 06/12 : 0 13 1 14 5 12 7 11 2 9 3 10 4 8 6 15\n", - "Mon Sep 6 11:45:58 2021[1,0]:dgx2h0194:41886:42730 [0] NCCL INFO Channel 07/12 : 0 13 1 14 5 12 7 11 2 9 3 10 4 8 6 15\n", - "Mon Sep 6 11:45:58 2021[1,15]:dgx2h0194:41901:42857 [0] NCCL INFO Trees [0] -1/-1/-1->15->6 [1] -1/-1/-1->15->6 [2] -1/-1/-1->15->6 [3] -1/-1/-1->15->6 [4] -1/-1/-1->15->6 [5] -1/-1/-1->15->6 [6] -1/-1/-1->15->6 [7] -1/-1/-1->15->6 [8] -1/-1/-1->15->6 [9] -1/-1/-1->15->6 [10] -1/-1/-1->15->6 [11] -1/-1/-1->15->6\n", - "Mon Sep 6 11:45:58 2021[1,15]:dgx2h0194:41901:42857 [0] NCCL INFO Setting affinity for GPU 9 to ffffff00,0000ffff,ff000000\n", - "Mon Sep 6 11:45:58 2021[1,2]:dgx2h0194:41888:42361 [0] NCCL INFO Trees [0] 9/-1/-1->2->11 [1] 9/-1/-1->2->11 [2] 9/-1/-1->2->11 [3] 9/-1/-1->2->11 [4] 9/-1/-1->2->11 [5] 9/-1/-1->2->11 [6] 9/-1/-1->2->11 [7] 9/-1/-1->2->11 [8] 9/-1/-1->2->11 [9] 9/-1/-1->2->11 [10] 9/-1/-1->2->11 [11] 9/-1/-1->2->11\n", - "Mon Sep 6 11:45:58 2021[1,2]:dgx2h0194:41888:42361 [0] NCCL INFO Setting affinity for GPU 14 to ffffff00,0000ffff,ff000000\n", - "Mon Sep 6 11:45:58 2021[1,1]:dgx2h0194:41887:42486 [0] NCCL INFO Trees [0] 14/-1/-1->1->13 [1] 14/-1/-1->1->13 [2] 14/-1/-1->1->13 [3] 14/-1/-1->1->13 [4] 14/-1/-1->1->13 [5] 14/-1/-1->1->13 [6] 14/-1/-1->1->13 [7] 14/-1/-1->1->13 [8] 14/-1/-1->1->13 [9] 14/-1/-1->1->13 [10] 14/-1/-1->1->13 [11] 14/-1/-1->1->13\n", - "Mon Sep 6 11:45:58 2021[1,1]:dgx2h0194:41887:42486 [0] NCCL INFO Setting affinity for GPU 2 to ff,ffff0000,00ffffff\n", - "Mon Sep 6 11:45:58 2021[1,3]:dgx2h0194:41889:42722 [0] NCCL INFO Trees [0] 10/-1/-1->3->9 [1] 10/-1/-1->3->9 [2] 10/-1/-1->3->9 [3] 10/-1/-1->3->9 [4] 10/-1/-1->3->9 [5] 10/-1/-1->3->9 [6] 10/-1/-1->3->9 [7] 10/-1/-1->3->9 [8] 10/-1/-1->3->9 [9] 10/-1/-1->3->9 [10] 10/-1/-1->3->9 [11] 10/-1/-1->3->9\n", - "Mon Sep 6 11:45:58 2021[1,3]:dgx2h0194:41889:42722 [0] NCCL INFO Setting affinity for GPU 13 to ffffff00,0000ffff,ff000000\n", - "Mon Sep 6 11:45:58 2021[1,14]:dgx2h0194:41900:42734 [0] NCCL INFO Trees [0] 5/-1/-1->14->1 [1] 5/-1/-1->14->1 [2] 5/-1/-1->14->1 [3] 5/-1/-1->14->1 [4] 5/-1/-1->14->1 [5] 5/-1/-1->14->1 [6] 5/-1/-1->14->1 [7] 5/-1/-1->14->1 [8] 5/-1/-1->14->1 [9] 5/-1/-1->14->1 [10] 5/-1/-1->14->1 [11] 5/-1/-1->14->1\n", - "Mon Sep 6 11:45:58 2021[1,14]:dgx2h0194:41900:42734 [0] NCCL INFO Setting affinity for GPU 3 to ff,ffff0000,00ffffff\n", - "Mon Sep 6 11:45:58 2021[1,0]:dgx2h0194:41886:42730 [0] NCCL INFO Channel 08/12 : 0 13 1 14 5 12 7 11 2 9 3 10 4 8 6 15\n", - "Mon Sep 6 11:45:58 2021[1,0]:dgx2h0194:41886:42730 [0] NCCL INFO Channel 09/12 : 0 13 1 14 5 12 7 11 2 9 3 10 4 8 6 15\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:45:58 2021[1,0]:dgx2h0194:41886:42730 [0] NCCL INFO Channel 10/12 : 0 13 1 14 5 12 7 11 2 9 3 10 4 8 6 15\n", - "Mon Sep 6 11:45:58 2021[1,0]:dgx2h0194:41886:42730 [0] NCCL INFO Channel 11/12 : 0 13 1 14 5 12 7 11 2 9 3 10 4 8 6 15\n", - "Mon Sep 6 11:45:58 2021[1,0]:dgx2h0194:41886:42730 [0] NCCL INFO Trees [0] 13/-1/-1->0->-1 [1] 13/-1/-1->0->-1 [2] 13/-1/-1->0->-1 [3] 13/-1/-1->0->-1 [4] 13/-1/-1->0->-1 [5] 13/-1/-1->0->-1 [6] 13/-1/-1->0->-1 [7] 13/-1/-1->0->-1 [8] 13/-1/-1->0->-1 [9] 13/-1/-1->0->-1 [10] 13/-1/-1->0->-1 [11] 13/-1/-1->0->-1\n", - "Mon Sep 6 11:45:58 2021[1,0]:dgx2h0194:41886:42730 [0] NCCL INFO Setting affinity for GPU 1 to ff,ffff0000,00ffffff\n", - "Mon Sep 6 11:45:58 2021[1,6]:dgx2h0194:41892:42733 [0] NCCL INFO Trees [0] 15/-1/-1->6->8 [1] 15/-1/-1->6->8 [2] 15/-1/-1->6->8 [3] 15/-1/-1->6->8 [4] 15/-1/-1->6->8 [5] 15/-1/-1->6->8 [6] 15/-1/-1->6->8 [7] 15/-1/-1->6->8 [8] 15/-1/-1->6->8 [9] 15/-1/-1->6->8 [10] 15/-1/-1->6->8 [11] 15/-1/-1->6->8\n", - "Mon Sep 6 11:45:58 2021[1,6]:dgx2h0194:41892:42733 [0] NCCL INFO Setting affinity for GPU 8 to ffffff00,0000ffff,ff000000\n", - "Mon Sep 6 11:45:58 2021[1,4]:dgx2h0194:41890:42739 [0] NCCL INFO Trees [0] 8/-1/-1->4->10 [1] 8/-1/-1->4->10 [2] 8/-1/-1->4->10 [3] 8/-1/-1->4->10 [4] 8/-1/-1->4->10 [5] 8/-1/-1->4->10 [6] 8/-1/-1->4->10 [7] 8/-1/-1->4->10 [8] 8/-1/-1->4->10 [9] 8/-1/-1->4->10 [10] 8/-1/-1->4->10 [11] 8/-1/-1->4->10\n", - "Mon Sep 6 11:45:58 2021[1,4]:dgx2h0194:41890:42739 [0] NCCL INFO Setting affinity for GPU 10 to ffffff00,0000ffff,ff000000\n", - "Mon Sep 6 11:45:58 2021[1,8]:dgx2h0194:41894:42489 [0] NCCL INFO Trees [0] 6/-1/-1->8->4 [1] 6/-1/-1->8->4 [2] 6/-1/-1->8->4 [3] 6/-1/-1->8->4 [4] 6/-1/-1->8->4 [5] 6/-1/-1->8->4 [6] 6/-1/-1->8->4 [7] 6/-1/-1->8->4 [8] 6/-1/-1->8->4 [9] 6/-1/-1->8->4 [10] 6/-1/-1->8->4 [11] 6/-1/-1->8->4\n", - "Mon Sep 6 11:45:58 2021[1,8]:dgx2h0194:41894:42489 [0] NCCL INFO Setting affinity for GPU 11 to ffffff00,0000ffff,ff000000\n", - "Mon Sep 6 11:45:58 2021[1,5]:dgx2h0194:41891:42723 [0] NCCL INFO Trees [0] 12/-1/-1->5->14 [1] 12/-1/-1->5->14 [2] 12/-1/-1->5->14 [3] 12/-1/-1->5->14 [4] 12/-1/-1->5->14 [5] 12/-1/-1->5->14 [6] 12/-1/-1->5->14 [7] 12/-1/-1->5->14 [8] 12/-1/-1->5->14 [9] 12/-1/-1->5->14 [10] 12/-1/-1->5->14 [11] 12/-1/-1->5->14\n", - "Mon Sep 6 11:45:58 2021[1,5]:dgx2h0194:41891:42723 [0] NCCL INFO Setting affinity for GPU 5 to ff,ffff0000,00ffffff\n", - "Mon Sep 6 11:45:58 2021[1,7]:dgx2h0194:41893:42743 [0] NCCL INFO Trees [0] 11/-1/-1->7->12 [1] 11/-1/-1->7->12 [2] 11/-1/-1->7->12 [3] 11/-1/-1->7->12 [4] 11/-1/-1->7->12 [5] 11/-1/-1->7->12 [6] 11/-1/-1->7->12 [7] 11/-1/-1->7->12 [8] 11/-1/-1->7->12 [9] 11/-1/-1->7->12 [10] 11/-1/-1->7->12 [11] 11/-1/-1->7->12\n", - "Mon Sep 6 11:45:58 2021[1,7]:dgx2h0194:41893:42743 [0] NCCL INFO Setting affinity for GPU 6 to ff,ffff0000,00ffffff\n", - "Mon Sep 6 11:45:58 2021[1,10]:dgx2h0194:41896:42859 [0] NCCL INFO Trees [0] 4/-1/-1->10->3 [1] 4/-1/-1->10->3 [2] 4/-1/-1->10->3 [3] 4/-1/-1->10->3 [4] 4/-1/-1->10->3 [5] 4/-1/-1->10->3 [6] 4/-1/-1->10->3 [7] 4/-1/-1->10->3 [8] 4/-1/-1->10->3 [9] 4/-1/-1->10->3 [10] 4/-1/-1->10->3 [11] 4/-1/-1->10->3\n", - "Mon Sep 6 11:45:58 2021[1,10]:dgx2h0194:41896:42859 [0] NCCL INFO Setting affinity for GPU 12 to ffffff00,0000ffff,ff000000\n", - "Mon Sep 6 11:45:58 2021[1,9]:dgx2h0194:41895:42860 [0] NCCL INFO Trees [0] 3/-1/-1->9->2 [1] 3/-1/-1->9->2 [2] 3/-1/-1->9->2 [3] 3/-1/-1->9->2 [4] 3/-1/-1->9->2 [5] 3/-1/-1->9->2 [6] 3/-1/-1->9->2 [7] 3/-1/-1->9->2 [8] 3/-1/-1->9->2 [9] 3/-1/-1->9->2 [10] 3/-1/-1->9->2 [11] 3/-1/-1->9->2\n", - "Mon Sep 6 11:45:58 2021[1,9]:dgx2h0194:41895:42860 [0] NCCL INFO Setting affinity for GPU 15 to ffffff00,0000ffff,ff000000\n", - "Mon Sep 6 11:45:58 2021[1,11]:dgx2h0194:41897:42858 [0] NCCL INFO Trees [0] 2/-1/-1->11->7 [1] 2/-1/-1->11->7 [2] 2/-1/-1->11->7 [3] 2/-1/-1->11->7 [4] 2/-1/-1->11->7 [5] 2/-1/-1->11->7 [6] 2/-1/-1->11->7 [7] 2/-1/-1->11->7 [8] 2/-1/-1->11->7 [9] 2/-1/-1->11->7 [10] 2/-1/-1->11->7 [11] 2/-1/-1->11->7\n", - "Mon Sep 6 11:45:58 2021[1,11]:dgx2h0194:41897:42858 [0] NCCL INFO Setting affinity for GPU 7 to ff,ffff0000,00ffffff\n", - "Mon Sep 6 11:45:58 2021[1,12]:dgx2h0194:41898:42413 [0] NCCL INFO Trees [0] 7/-1/-1->12->5 [1] 7/-1/-1->12->5 [2] 7/-1/-1->12->5 [3] 7/-1/-1->12->5 [4] 7/-1/-1->12->5 [5] 7/-1/-1->12->5 [6] 7/-1/-1->12->5 [7] 7/-1/-1->12->5 [8] 7/-1/-1->12->5 [9] 7/-1/-1->12->5 [10] 7/-1/-1->12->5 [11] 7/-1/-1->12->5\n", - "Mon Sep 6 11:45:58 2021[1,12]:dgx2h0194:41898:42413 [0] NCCL INFO Setting affinity for GPU 4 to ff,ffff0000,00ffffff\n", - "Mon Sep 6 11:45:58 2021[1,13]:dgx2h0194:41899:42721 [0] NCCL INFO Channel 00 : 13[34000] -> 1[39000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,14]:dgx2h0194:41900:42734 [0] NCCL INFO Channel 00 : 14[3b000] -> 5[59000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,7]:dgx2h0194:41893:42743 [0] NCCL INFO Channel 00 : 7[5c000] -> 11[5e000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,15]:dgx2h0194:41901:42857 [0] NCCL INFO Channel 00 : 15[b9000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,4]:dgx2h0194:41890:42739 [0] NCCL INFO Channel 00 : 4[bc000] -> 8[be000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,3]:dgx2h0194:41889:42722 [0] NCCL INFO Channel 00 : 3[e2000] -> 10[e0000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,13]:dgx2h0194:41899:42721 [0] NCCL INFO Channel 01 : 13[34000] -> 1[39000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,6]:dgx2h0194:41892:42733 [0] NCCL INFO Channel 00 : 6[b7000] -> 15[b9000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,14]:dgx2h0194:41900:42734 [0] NCCL INFO Channel 01 : 14[3b000] -> 5[59000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,7]:dgx2h0194:41893:42743 [0] NCCL INFO Channel 01 : 7[5c000] -> 11[5e000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,15]:dgx2h0194:41901:42857 [0] NCCL INFO Channel 01 : 15[b9000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,4]:dgx2h0194:41890:42739 [0] NCCL INFO Channel 01 : 4[bc000] -> 8[be000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,3]:dgx2h0194:41889:42722 [0] NCCL INFO Channel 01 : 3[e2000] -> 10[e0000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,6]:dgx2h0194:41892:42733 [0] NCCL INFO Channel 01 : 6[b7000] -> 15[b9000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,13]:dgx2h0194:41899:42721 [0] NCCL INFO Channel 02 : 13[34000] -> 1[39000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,14]:dgx2h0194:41900:42734 [0] NCCL INFO Channel 02 : 14[3b000] -> 5[59000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,7]:dgx2h0194:41893:42743 [0] NCCL INFO Channel 02 : 7[5c000] -> 11[5e000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,15]:dgx2h0194:41901:42857 [0] NCCL INFO Channel 02 : 15[b9000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,4]:dgx2h0194:41890:42739 [0] NCCL INFO Channel 02 : 4[bc000] -> 8[be000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,3]:dgx2h0194:41889:42722 [0] NCCL INFO Channel 02 : 3[e2000] -> 10[e0000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,13]:dgx2h0194:41899:42721 [0] NCCL INFO Channel 03 : 13[34000] -> 1[39000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,6]:dgx2h0194:41892:42733 [0] NCCL INFO Channel 02 : 6[b7000] -> 15[b9000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,14]:dgx2h0194:41900:42734 [0] NCCL INFO Channel 03 : 14[3b000] -> 5[59000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,7]:dgx2h0194:41893:42743 [0] NCCL INFO Channel 03 : 7[5c000] -> 11[5e000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,15]:dgx2h0194:41901:42857 [0] NCCL INFO Channel 03 : 15[b9000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,4]:dgx2h0194:41890:42739 [0] NCCL INFO Channel 03 : 4[bc000] -> 8[be000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,3]:dgx2h0194:41889:42722 [0] NCCL INFO Channel 03 : 3[e2000] -> 10[e0000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,6]:dgx2h0194:41892:42733 [0] NCCL INFO Channel 03 : 6[b7000] -> 15[b9000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,13]:dgx2h0194:41899:42721 [0] NCCL INFO Channel 04 : 13[34000] -> 1[39000] via P2P/IPC\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:45:58 2021[1,14]:dgx2h0194:41900:42734 [0] NCCL INFO Channel 04 : 14[3b000] -> 5[59000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,7]:dgx2h0194:41893:42743 [0] NCCL INFO Channel 04 : 7[5c000] -> 11[5e000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,15]:dgx2h0194:41901:42857 [0] NCCL INFO Channel 04 : 15[b9000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,4]:dgx2h0194:41890:42739 [0] NCCL INFO Channel 04 : 4[bc000] -> 8[be000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,3]:dgx2h0194:41889:42722 [0] NCCL INFO Channel 04 : 3[e2000] -> 10[e0000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,6]:dgx2h0194:41892:42733 [0] NCCL INFO Channel 04 : 6[b7000] -> 15[b9000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,13]:dgx2h0194:41899:42721 [0] NCCL INFO Channel 05 : 13[34000] -> 1[39000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,14]:dgx2h0194:41900:42734 [0] NCCL INFO Channel 05 : 14[3b000] -> 5[59000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,7]:dgx2h0194:41893:42743 [0] NCCL INFO Channel 05 : 7[5c000] -> 11[5e000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,15]:dgx2h0194:41901:42857 [0] NCCL INFO Channel 05 : 15[b9000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,4]:dgx2h0194:41890:42739 [0] NCCL INFO Channel 05 : 4[bc000] -> 8[be000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,3]:dgx2h0194:41889:42722 [0] NCCL INFO Channel 05 : 3[e2000] -> 10[e0000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,13]:dgx2h0194:41899:42721 [0] NCCL INFO Channel 06 : 13[34000] -> 1[39000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,6]:dgx2h0194:41892:42733 [0] NCCL INFO Channel 05 : 6[b7000] -> 15[b9000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,14]:dgx2h0194:41900:42734 [0] NCCL INFO Channel 06 : 14[3b000] -> 5[59000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,7]:dgx2h0194:41893:42743 [0] NCCL INFO Channel 06 : 7[5c000] -> 11[5e000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,15]:dgx2h0194:41901:42857 [0] NCCL INFO Channel 06 : 15[b9000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,4]:dgx2h0194:41890:42739 [0] NCCL INFO Channel 06 : 4[bc000] -> 8[be000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,3]:dgx2h0194:41889:42722 [0] NCCL INFO Channel 06 : 3[e2000] -> 10[e0000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,6]:dgx2h0194:41892:42733 [0] NCCL INFO Channel 06 : 6[b7000] -> 15[b9000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,13]:dgx2h0194:41899:42721 [0] NCCL INFO Channel 07 : 13[34000] -> 1[39000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,14]:dgx2h0194:41900:42734 [0] NCCL INFO Channel 07 : 14[3b000] -> 5[59000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,7]:dgx2h0194:41893:42743 [0] NCCL INFO Channel 07 : 7[5c000] -> 11[5e000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,15]:dgx2h0194:41901:42857 [0] NCCL INFO Channel 07 : 15[b9000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,4]:dgx2h0194:41890:42739 [0] NCCL INFO Channel 07 : 4[bc000] -> 8[be000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,3]:dgx2h0194:41889:42722 [0] NCCL INFO Channel 07 : 3[e2000] -> 10[e0000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,13]:dgx2h0194:41899:42721 [0] NCCL INFO Channel 08 : 13[34000] -> 1[39000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,6]:dgx2h0194:41892:42733 [0] NCCL INFO Channel 07 : 6[b7000] -> 15[b9000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,14]:dgx2h0194:41900:42734 [0] NCCL INFO Channel 08 : 14[3b000] -> 5[59000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,7]:dgx2h0194:41893:42743 [0] NCCL INFO Channel 08 : 7[5c000] -> 11[5e000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,15]:dgx2h0194:41901:42857 [0] NCCL INFO Channel 08 : 15[b9000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,4]:dgx2h0194:41890:42739 [0] NCCL INFO Channel 08 : 4[bc000] -> 8[be000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,3]:dgx2h0194:41889:42722 [0] NCCL INFO Channel 08 : 3[e2000] -> 10[e0000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,6]:dgx2h0194:41892:42733 [0] NCCL INFO Channel 08 : 6[b7000] -> 15[b9000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,13]:dgx2h0194:41899:42721 [0] NCCL INFO Channel 09 : 13[34000] -> 1[39000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,14]:dgx2h0194:41900:42734 [0] NCCL INFO Channel 09 : 14[3b000] -> 5[59000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,7]:dgx2h0194:41893:42743 [0] NCCL INFO Channel 09 : 7[5c000] -> 11[5e000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,15]:dgx2h0194:41901:42857 [0] NCCL INFO Channel 09 : 15[b9000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,4]:dgx2h0194:41890:42739 [0] NCCL INFO Channel 09 : 4[bc000] -> 8[be000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,3]:dgx2h0194:41889:42722 [0] NCCL INFO Channel 09 : 3[e2000] -> 10[e0000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,13]:dgx2h0194:41899:42721 [0] NCCL INFO Channel 10 : 13[34000] -> 1[39000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,6]:dgx2h0194:41892:42733 [0] NCCL INFO Channel 09 : 6[b7000] -> 15[b9000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,14]:dgx2h0194:41900:42734 [0] NCCL INFO Channel 10 : 14[3b000] -> 5[59000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,7]:dgx2h0194:41893:42743 [0] NCCL INFO Channel 10 : 7[5c000] -> 11[5e000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,15]:dgx2h0194:41901:42857 [0] NCCL INFO Channel 10 : 15[b9000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,4]:dgx2h0194:41890:42739 [0] NCCL INFO Channel 10 : 4[bc000] -> 8[be000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,3]:dgx2h0194:41889:42722 [0] NCCL INFO Channel 10 : 3[e2000] -> 10[e0000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,6]:dgx2h0194:41892:42733 [0] NCCL INFO Channel 10 : 6[b7000] -> 15[b9000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,13]:dgx2h0194:41899:42721 [0] NCCL INFO Channel 11 : 13[34000] -> 1[39000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,14]:dgx2h0194:41900:42734 [0] NCCL INFO Channel 11 : 14[3b000] -> 5[59000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,7]:dgx2h0194:41893:42743 [0] NCCL INFO Channel 11 : 7[5c000] -> 11[5e000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,15]:dgx2h0194:41901:42857 [0] NCCL INFO Channel 11 : 15[b9000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,4]:dgx2h0194:41890:42739 [0] NCCL INFO Channel 11 : 4[bc000] -> 8[be000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,3]:dgx2h0194:41889:42722 [0] NCCL INFO Channel 11 : 3[e2000] -> 10[e0000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,6]:dgx2h0194:41892:42733 [0] NCCL INFO Channel 11 : 6[b7000] -> 15[b9000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,5]:dgx2h0194:41891:42723 [0] NCCL INFO Channel 00 : 5[59000] -> 12[57000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,2]:dgx2h0194:41888:42361 [0] NCCL INFO Channel 00 : 2[e5000] -> 9[e7000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,5]:dgx2h0194:41891:42723 [0] NCCL INFO Channel 01 : 5[59000] -> 12[57000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,2]:dgx2h0194:41888:42361 [0] NCCL INFO Channel 01 : 2[e5000] -> 9[e7000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,5]:dgx2h0194:41891:42723 [0] NCCL INFO Channel 02 : 5[59000] -> 12[57000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,2]:dgx2h0194:41888:42361 [0] NCCL INFO Channel 02 : 2[e5000] -> 9[e7000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,5]:dgx2h0194:41891:42723 [0] NCCL INFO Channel 03 : 5[59000] -> 12[57000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,5]:dgx2h0194:41891:42723 [0] NCCL INFO Channel 04 : 5[59000] -> 12[57000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,2]:dgx2h0194:41888:42361 [0] NCCL INFO Channel 03 : 2[e5000] -> 9[e7000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,5]:dgx2h0194:41891:42723 [0] NCCL INFO Channel 05 : 5[59000] -> 12[57000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,2]:dgx2h0194:41888:42361 [0] NCCL INFO Channel 04 : 2[e5000] -> 9[e7000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,5]:dgx2h0194:41891:42723 [0] NCCL INFO Channel 06 : 5[59000] -> 12[57000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,2]:dgx2h0194:41888:42361 [0] NCCL INFO Channel 05 : 2[e5000] -> 9[e7000] via P2P/IPC\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:45:58 2021[1,5]:dgx2h0194:41891:42723 [0] NCCL INFO Channel 07 : 5[59000] -> 12[57000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,2]:dgx2h0194:41888:42361 [0] NCCL INFO Channel 06 : 2[e5000] -> 9[e7000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,5]:dgx2h0194:41891:42723 [0] NCCL INFO Channel 08 : 5[59000] -> 12[57000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,2]:dgx2h0194:41888:42361 [0] NCCL INFO Channel 07 : 2[e5000] -> 9[e7000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,5]:dgx2h0194:41891:42723 [0] NCCL INFO Channel 09 : 5[59000] -> 12[57000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,2]:dgx2h0194:41888:42361 [0] NCCL INFO Channel 08 : 2[e5000] -> 9[e7000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,5]:dgx2h0194:41891:42723 [0] NCCL INFO Channel 10 : 5[59000] -> 12[57000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,2]:dgx2h0194:41888:42361 [0] NCCL INFO Channel 09 : 2[e5000] -> 9[e7000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,5]:dgx2h0194:41891:42723 [0] NCCL INFO Channel 11 : 5[59000] -> 12[57000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,2]:dgx2h0194:41888:42361 [0] NCCL INFO Channel 10 : 2[e5000] -> 9[e7000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,2]:dgx2h0194:41888:42361 [0] NCCL INFO Channel 11 : 2[e5000] -> 9[e7000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,1]:dgx2h0194:41887:42486 [0] NCCL INFO Channel 00 : 1[39000] -> 14[3b000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,1]:dgx2h0194:41887:42486 [0] NCCL INFO Channel 01 : 1[39000] -> 14[3b000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,1]:dgx2h0194:41887:42486 [0] NCCL INFO Channel 02 : 1[39000] -> 14[3b000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,1]:dgx2h0194:41887:42486 [0] NCCL INFO Channel 03 : 1[39000] -> 14[3b000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,1]:dgx2h0194:41887:42486 [0] NCCL INFO Channel 04 : 1[39000] -> 14[3b000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,1]:dgx2h0194:41887:42486 [0] NCCL INFO Channel 05 : 1[39000] -> 14[3b000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,1]:dgx2h0194:41887:42486 [0] NCCL INFO Channel 06 : 1[39000] -> 14[3b000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,1]:dgx2h0194:41887:42486 [0] NCCL INFO Channel 07 : 1[39000] -> 14[3b000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,1]:dgx2h0194:41887:42486 [0] NCCL INFO Channel 08 : 1[39000] -> 14[3b000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,1]:dgx2h0194:41887:42486 [0] NCCL INFO Channel 09 : 1[39000] -> 14[3b000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,11]:dgx2h0194:41897:42858 [0] NCCL INFO Channel 00 : 11[5e000] -> 2[e5000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,1]:dgx2h0194:41887:42486 [0] NCCL INFO Channel 10 : 1[39000] -> 14[3b000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,11]:dgx2h0194:41897:42858 [0] NCCL INFO Channel 01 : 11[5e000] -> 2[e5000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,1]:dgx2h0194:41887:42486 [0] NCCL INFO Channel 11 : 1[39000] -> 14[3b000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,11]:dgx2h0194:41897:42858 [0] NCCL INFO Channel 02 : 11[5e000] -> 2[e5000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,11]:dgx2h0194:41897:42858 [0] NCCL INFO Channel 03 : 11[5e000] -> 2[e5000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,11]:dgx2h0194:41897:42858 [0] NCCL INFO Channel 04 : 11[5e000] -> 2[e5000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,11]:dgx2h0194:41897:42858 [0] NCCL INFO Channel 05 : 11[5e000] -> 2[e5000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,11]:dgx2h0194:41897:42858 [0] NCCL INFO Channel 06 : 11[5e000] -> 2[e5000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,0]:dgx2h0194:41886:42730 [0] NCCL INFO Channel 00 : 0[36000] -> 13[34000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,11]:dgx2h0194:41897:42858 [0] NCCL INFO Channel 07 : 11[5e000] -> 2[e5000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,0]:dgx2h0194:41886:42730 [0] NCCL INFO Channel 01 : 0[36000] -> 13[34000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,11]:dgx2h0194:41897:42858 [0] NCCL INFO Channel 08 : 11[5e000] -> 2[e5000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,0]:dgx2h0194:41886:42730 [0] NCCL INFO Channel 02 : 0[36000] -> 13[34000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,11]:dgx2h0194:41897:42858 [0] NCCL INFO Channel 09 : 11[5e000] -> 2[e5000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,0]:dgx2h0194:41886:42730 [0] NCCL INFO Channel 03 : 0[36000] -> 13[34000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,8]:dgx2h0194:41894:42489 [0] NCCL INFO Channel 00 : 8[be000] -> 6[b7000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,11]:dgx2h0194:41897:42858 [0] NCCL INFO Channel 10 : 11[5e000] -> 2[e5000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,0]:dgx2h0194:41886:42730 [0] NCCL INFO Channel 04 : 0[36000] -> 13[34000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,8]:dgx2h0194:41894:42489 [0] NCCL INFO Channel 01 : 8[be000] -> 6[b7000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,11]:dgx2h0194:41897:42858 [0] NCCL INFO Channel 11 : 11[5e000] -> 2[e5000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,0]:dgx2h0194:41886:42730 [0] NCCL INFO Channel 05 : 0[36000] -> 13[34000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,8]:dgx2h0194:41894:42489 [0] NCCL INFO Channel 02 : 8[be000] -> 6[b7000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,0]:dgx2h0194:41886:42730 [0] NCCL INFO Channel 06 : 0[36000] -> 13[34000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,8]:dgx2h0194:41894:42489 [0] NCCL INFO Channel 03 : 8[be000] -> 6[b7000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,0]:dgx2h0194:41886:42730 [0] NCCL INFO Channel 07 : 0[36000] -> 13[34000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,8]:dgx2h0194:41894:42489 [0] NCCL INFO Channel 04 : 8[be000] -> 6[b7000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,0]:dgx2h0194:41886:42730 [0] NCCL INFO Channel 08 : 0[36000] -> 13[34000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,8]:dgx2h0194:41894:42489 [0] NCCL INFO Channel 05 : 8[be000] -> 6[b7000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,0]:dgx2h0194:41886:42730 [0] NCCL INFO Channel 09 : 0[36000] -> 13[34000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,8]:dgx2h0194:41894:42489 [0] NCCL INFO Channel 06 : 8[be000] -> 6[b7000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,0]:dgx2h0194:41886:42730 [0] NCCL INFO Channel 10 : 0[36000] -> 13[34000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,8]:dgx2h0194:41894:42489 [0] NCCL INFO Channel 07 : 8[be000] -> 6[b7000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,0]:dgx2h0194:41886:42730 [0] NCCL INFO Channel 11 : 0[36000] -> 13[34000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,8]:dgx2h0194:41894:42489 [0] NCCL INFO Channel 08 : 8[be000] -> 6[b7000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,8]:dgx2h0194:41894:42489 [0] NCCL INFO Channel 09 : 8[be000] -> 6[b7000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,10]:dgx2h0194:41896:42859 [0] NCCL INFO Channel 00 : 10[e0000] -> 4[bc000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,8]:dgx2h0194:41894:42489 [0] NCCL INFO Channel 10 : 8[be000] -> 6[b7000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,10]:dgx2h0194:41896:42859 [0] NCCL INFO Channel 01 : 10[e0000] -> 4[bc000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,8]:dgx2h0194:41894:42489 [0] NCCL INFO Channel 11 : 8[be000] -> 6[b7000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,10]:dgx2h0194:41896:42859 [0] NCCL INFO Channel 02 : 10[e0000] -> 4[bc000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,10]:dgx2h0194:41896:42859 [0] NCCL INFO Channel 03 : 10[e0000] -> 4[bc000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,10]:dgx2h0194:41896:42859 [0] NCCL INFO Channel 04 : 10[e0000] -> 4[bc000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,10]:dgx2h0194:41896:42859 [0] NCCL INFO Channel 05 : 10[e0000] -> 4[bc000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,10]:dgx2h0194:41896:42859 [0] NCCL INFO Channel 06 : 10[e0000] -> 4[bc000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,10]:dgx2h0194:41896:42859 [0] NCCL INFO Channel 07 : 10[e0000] -> 4[bc000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,10]:dgx2h0194:41896:42859 [0] NCCL INFO Channel 08 : 10[e0000] -> 4[bc000] via P2P/IPC\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:45:58 2021[1,10]:dgx2h0194:41896:42859 [0] NCCL INFO Channel 09 : 10[e0000] -> 4[bc000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,12]:dgx2h0194:41898:42413 [0] NCCL INFO Channel 00 : 12[57000] -> 7[5c000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,10]:dgx2h0194:41896:42859 [0] NCCL INFO Channel 10 : 10[e0000] -> 4[bc000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,12]:dgx2h0194:41898:42413 [0] NCCL INFO Channel 01 : 12[57000] -> 7[5c000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,10]:dgx2h0194:41896:42859 [0] NCCL INFO Channel 11 : 10[e0000] -> 4[bc000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,12]:dgx2h0194:41898:42413 [0] NCCL INFO Channel 02 : 12[57000] -> 7[5c000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,12]:dgx2h0194:41898:42413 [0] NCCL INFO Channel 03 : 12[57000] -> 7[5c000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,9]:dgx2h0194:41895:42860 [0] NCCL INFO Channel 00 : 9[e7000] -> 3[e2000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,12]:dgx2h0194:41898:42413 [0] NCCL INFO Channel 04 : 12[57000] -> 7[5c000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,9]:dgx2h0194:41895:42860 [0] NCCL INFO Channel 01 : 9[e7000] -> 3[e2000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,12]:dgx2h0194:41898:42413 [0] NCCL INFO Channel 05 : 12[57000] -> 7[5c000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,9]:dgx2h0194:41895:42860 [0] NCCL INFO Channel 02 : 9[e7000] -> 3[e2000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,15]:dgx2h0194:41901:42857 [0] NCCL INFO Connected all rings\n", - "Mon Sep 6 11:45:58 2021[1,12]:dgx2h0194:41898:42413 [0] NCCL INFO Channel 06 : 12[57000] -> 7[5c000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,9]:dgx2h0194:41895:42860 [0] NCCL INFO Channel 03 : 9[e7000] -> 3[e2000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,15]:dgx2h0194:41901:42857 [0] NCCL INFO Channel 00 : 15[b9000] -> 6[b7000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,12]:dgx2h0194:41898:42413 [0] NCCL INFO Channel 07 : 12[57000] -> 7[5c000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,9]:dgx2h0194:41895:42860 [0] NCCL INFO Channel 04 : 9[e7000] -> 3[e2000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,15]:dgx2h0194:41901:42857 [0] NCCL INFO Channel 01 : 15[b9000] -> 6[b7000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,12]:dgx2h0194:41898:42413 [0] NCCL INFO Channel 08 : 12[57000] -> 7[5c000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,9]:dgx2h0194:41895:42860 [0] NCCL INFO Channel 05 : 9[e7000] -> 3[e2000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,15]:dgx2h0194:41901:42857 [0] NCCL INFO Channel 02 : 15[b9000] -> 6[b7000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,9]:dgx2h0194:41895:42860 [0] NCCL INFO Channel 06 : 9[e7000] -> 3[e2000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,12]:dgx2h0194:41898:42413 [0] NCCL INFO Channel 09 : 12[57000] -> 7[5c000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,15]:dgx2h0194:41901:42857 [0] NCCL INFO Channel 03 : 15[b9000] -> 6[b7000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,12]:dgx2h0194:41898:42413 [0] NCCL INFO Channel 10 : 12[57000] -> 7[5c000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,9]:dgx2h0194:41895:42860 [0] NCCL INFO Channel 07 : 9[e7000] -> 3[e2000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,15]:dgx2h0194:41901:42857 [0] NCCL INFO Channel 04 : 15[b9000] -> 6[b7000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,12]:dgx2h0194:41898:42413 [0] NCCL INFO Channel 11 : 12[57000] -> 7[5c000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,9]:dgx2h0194:41895:42860 [0] NCCL INFO Channel 08 : 9[e7000] -> 3[e2000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,15]:dgx2h0194:41901:42857 [0] NCCL INFO Channel 05 : 15[b9000] -> 6[b7000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,11]:dgx2h0194:41897:42858 [0] NCCL INFO Connected all rings\n", - "Mon Sep 6 11:45:58 2021[1,9]:dgx2h0194:41895:42860 [0] NCCL INFO Channel 09 : 9[e7000] -> 3[e2000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,15]:dgx2h0194:41901:42857 [0] NCCL INFO Channel 06 : 15[b9000] -> 6[b7000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,9]:dgx2h0194:41895:42860 [0] NCCL INFO Channel 10 : 9[e7000] -> 3[e2000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,15]:dgx2h0194:41901:42857 [0] NCCL INFO Channel 07 : 15[b9000] -> 6[b7000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,9]:dgx2h0194:41895:42860 [0] NCCL INFO Channel 11 : 9[e7000] -> 3[e2000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,15]:dgx2h0194:41901:42857 [0] NCCL INFO Channel 08 : 15[b9000] -> 6[b7000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,0]:dgx2h0194:41886:42730 [0] NCCL INFO Connected all rings\n", - "Mon Sep 6 11:45:58 2021[1,15]:dgx2h0194:41901:42857 [0] NCCL INFO Channel 09 : 15[b9000] -> 6[b7000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,15]:dgx2h0194:41901:42857 [0] NCCL INFO Channel 10 : 15[b9000] -> 6[b7000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,15]:dgx2h0194:41901:42857 [0] NCCL INFO Channel 11 : 15[b9000] -> 6[b7000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,13]:dgx2h0194:41899:42721 [0] NCCL INFO Connected all rings\n", - "Mon Sep 6 11:45:58 2021[1,13]:dgx2h0194:41899:42721 [0] NCCL INFO Channel 00 : 13[34000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 11:45:58 2021[1,13]:dgx2h0194:41899:42721 [0] NCCL INFO Channel 01 : 13[34000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,13]:dgx2h0194:41899:42721 [0] NCCL INFO Channel 02 : 13[34000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,13]:dgx2h0194:41899:42721 [0] NCCL INFO Channel 03 : 13[34000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,13]:dgx2h0194:41899:42721 [0] NCCL INFO Channel 04 : 13[34000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,13]:dgx2h0194:41899:42721 [0] NCCL INFO Channel 05 : 13[34000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,5]:dgx2h0194:41891:42723 [0] NCCL INFO Connected all rings\n", - "Mon Sep 6 11:45:59 2021[1,13]:dgx2h0194:41899:42721 [0] NCCL INFO Channel 06 : 13[34000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,13]:dgx2h0194:41899:42721 [0] NCCL INFO Channel 07 : 13[34000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,13]:dgx2h0194:41899:42721 [0] NCCL INFO Channel 08 : 13[34000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,10]:dgx2h0194:41896:42859 [0] NCCL INFO Connected all rings\n", - "Mon Sep 6 11:45:59 2021[1,13]:dgx2h0194:41899:42721 [0] NCCL INFO Channel 09 : 13[34000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,13]:dgx2h0194:41899:42721 [0] NCCL INFO Channel 10 : 13[34000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,13]:dgx2h0194:41899:42721 [0] NCCL INFO Channel 11 : 13[34000] -> 0[36000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,4]:dgx2h0194:41890:42739 [0] NCCL INFO Connected all rings\n", - "Mon Sep 6 11:45:59 2021[1,4]:dgx2h0194:41890:42739 [0] NCCL INFO Channel 00 : 4[bc000] -> 10[e0000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,4]:dgx2h0194:41890:42739 [0] NCCL INFO Channel 01 : 4[bc000] -> 10[e0000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,4]:dgx2h0194:41890:42739 [0] NCCL INFO Channel 02 : 4[bc000] -> 10[e0000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,14]:dgx2h0194:41900:42734 [0] NCCL INFO Connected all rings\n", - "Mon Sep 6 11:45:59 2021[1,4]:dgx2h0194:41890:42739 [0] NCCL INFO Channel 03 : 4[bc000] -> 10[e0000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,14]:dgx2h0194:41900:42734 [0] NCCL INFO Channel 00 : 14[3b000] -> 1[39000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,4]:dgx2h0194:41890:42739 [0] NCCL INFO Channel 04 : 4[bc000] -> 10[e0000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,14]:dgx2h0194:41900:42734 [0] NCCL INFO Channel 01 : 14[3b000] -> 1[39000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,1]:dgx2h0194:41887:42486 [0] NCCL INFO Connected all rings\n", - "Mon Sep 6 11:45:59 2021[1,4]:dgx2h0194:41890:42739 [0] NCCL INFO Channel 05 : 4[bc000] -> 10[e0000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,14]:dgx2h0194:41900:42734 [0] NCCL INFO Channel 02 : 14[3b000] -> 1[39000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,5]:dgx2h0194:41891:42723 [0] NCCL INFO Channel 00 : 5[59000] -> 14[3b000] via P2P/IPC\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:45:59 2021[1,0]:dgx2h0194:41886:42730 [0] NCCL INFO Connected all trees\n", - "Mon Sep 6 11:45:59 2021[1,0]:dgx2h0194:41886:42730 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 8/8/512\n", - "Mon Sep 6 11:45:59 2021[1,4]:dgx2h0194:41890:42739 [0] NCCL INFO Channel 06 : 4[bc000] -> 10[e0000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,14]:dgx2h0194:41900:42734 [0] NCCL INFO Channel 03 : 14[3b000] -> 1[39000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,5]:dgx2h0194:41891:42723 [0] NCCL INFO Channel 01 : 5[59000] -> 14[3b000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,4]:dgx2h0194:41890:42739 [0] NCCL INFO Channel 07 : 4[bc000] -> 10[e0000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,14]:dgx2h0194:41900:42734 [0] NCCL INFO Channel 04 : 14[3b000] -> 1[39000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,0]:dgx2h0194:41886:42730 [0] NCCL INFO 12 coll channels, 16 p2p channels, 16 p2p channels per peer\n", - "Mon Sep 6 11:45:59 2021[1,5]:dgx2h0194:41891:42723 [0] NCCL INFO Channel 02 : 5[59000] -> 14[3b000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,4]:dgx2h0194:41890:42739 [0] NCCL INFO Channel 08 : 4[bc000] -> 10[e0000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,14]:dgx2h0194:41900:42734 [0] NCCL INFO Channel 05 : 14[3b000] -> 1[39000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,7]:dgx2h0194:41893:42743 [0] NCCL INFO Connected all rings\n", - "Mon Sep 6 11:45:59 2021[1,5]:dgx2h0194:41891:42723 [0] NCCL INFO Channel 03 : 5[59000] -> 14[3b000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,4]:dgx2h0194:41890:42739 [0] NCCL INFO Channel 09 : 4[bc000] -> 10[e0000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,14]:dgx2h0194:41900:42734 [0] NCCL INFO Channel 06 : 14[3b000] -> 1[39000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,7]:dgx2h0194:41893:42743 [0] NCCL INFO Channel 00 : 7[5c000] -> 12[57000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,5]:dgx2h0194:41891:42723 [0] NCCL INFO Channel 04 : 5[59000] -> 14[3b000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,4]:dgx2h0194:41890:42739 [0] NCCL INFO Channel 10 : 4[bc000] -> 10[e0000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,7]:dgx2h0194:41893:42743 [0] NCCL INFO Channel 01 : 7[5c000] -> 12[57000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,14]:dgx2h0194:41900:42734 [0] NCCL INFO Channel 07 : 14[3b000] -> 1[39000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,5]:dgx2h0194:41891:42723 [0] NCCL INFO Channel 05 : 5[59000] -> 14[3b000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,4]:dgx2h0194:41890:42739 [0] NCCL INFO Channel 11 : 4[bc000] -> 10[e0000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,14]:dgx2h0194:41900:42734 [0] NCCL INFO Channel 08 : 14[3b000] -> 1[39000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,7]:dgx2h0194:41893:42743 [0] NCCL INFO Channel 02 : 7[5c000] -> 12[57000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,12]:dgx2h0194:41898:42413 [0] NCCL INFO Connected all rings\n", - "Mon Sep 6 11:45:59 2021[1,5]:dgx2h0194:41891:42723 [0] NCCL INFO Channel 06 : 5[59000] -> 14[3b000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,7]:dgx2h0194:41893:42743 [0] NCCL INFO Channel 03 : 7[5c000] -> 12[57000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,14]:dgx2h0194:41900:42734 [0] NCCL INFO Channel 09 : 14[3b000] -> 1[39000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,5]:dgx2h0194:41891:42723 [0] NCCL INFO Channel 07 : 5[59000] -> 14[3b000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,14]:dgx2h0194:41900:42734 [0] NCCL INFO Channel 10 : 14[3b000] -> 1[39000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,7]:dgx2h0194:41893:42743 [0] NCCL INFO Channel 04 : 7[5c000] -> 12[57000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,5]:dgx2h0194:41891:42723 [0] NCCL INFO Channel 08 : 5[59000] -> 14[3b000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,14]:dgx2h0194:41900:42734 [0] NCCL INFO Channel 11 : 14[3b000] -> 1[39000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,7]:dgx2h0194:41893:42743 [0] NCCL INFO Channel 05 : 7[5c000] -> 12[57000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,9]:dgx2h0194:41895:42860 [0] NCCL INFO Connected all rings\n", - "Mon Sep 6 11:45:59 2021[1,5]:dgx2h0194:41891:42723 [0] NCCL INFO Channel 09 : 5[59000] -> 14[3b000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,7]:dgx2h0194:41893:42743 [0] NCCL INFO Channel 06 : 7[5c000] -> 12[57000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,5]:dgx2h0194:41891:42723 [0] NCCL INFO Channel 10 : 5[59000] -> 14[3b000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,7]:dgx2h0194:41893:42743 [0] NCCL INFO Channel 07 : 7[5c000] -> 12[57000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,5]:dgx2h0194:41891:42723 [0] NCCL INFO Channel 11 : 5[59000] -> 14[3b000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,7]:dgx2h0194:41893:42743 [0] NCCL INFO Channel 08 : 7[5c000] -> 12[57000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,3]:dgx2h0194:41889:42722 [0] NCCL INFO Connected all rings\n", - "Mon Sep 6 11:45:59 2021[1,7]:dgx2h0194:41893:42743 [0] NCCL INFO Channel 09 : 7[5c000] -> 12[57000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,3]:dgx2h0194:41889:42722 [0] NCCL INFO Channel 00 : 3[e2000] -> 9[e7000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,7]:dgx2h0194:41893:42743 [0] NCCL INFO Channel 10 : 7[5c000] -> 12[57000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,10]:dgx2h0194:41896:42859 [0] NCCL INFO Channel 00 : 10[e0000] -> 3[e2000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,3]:dgx2h0194:41889:42722 [0] NCCL INFO Channel 01 : 3[e2000] -> 9[e7000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,7]:dgx2h0194:41893:42743 [0] NCCL INFO Channel 11 : 7[5c000] -> 12[57000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,3]:dgx2h0194:41889:42722 [0] NCCL INFO Channel 02 : 3[e2000] -> 9[e7000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,10]:dgx2h0194:41896:42859 [0] NCCL INFO Channel 01 : 10[e0000] -> 3[e2000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,8]:dgx2h0194:41894:42489 [0] NCCL INFO Connected all rings\n", - "Mon Sep 6 11:45:59 2021[1,10]:dgx2h0194:41896:42859 [0] NCCL INFO Channel 02 : 10[e0000] -> 3[e2000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,3]:dgx2h0194:41889:42722 [0] NCCL INFO Channel 03 : 3[e2000] -> 9[e7000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,10]:dgx2h0194:41896:42859 [0] NCCL INFO Channel 03 : 10[e0000] -> 3[e2000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,3]:dgx2h0194:41889:42722 [0] NCCL INFO Channel 04 : 3[e2000] -> 9[e7000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,3]:dgx2h0194:41889:42722 [0] NCCL INFO Channel 05 : 3[e2000] -> 9[e7000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,10]:dgx2h0194:41896:42859 [0] NCCL INFO Channel 04 : 10[e0000] -> 3[e2000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,2]:dgx2h0194:41888:42361 [0] NCCL INFO Connected all rings\n", - "Mon Sep 6 11:45:59 2021[1,3]:dgx2h0194:41889:42722 [0] NCCL INFO Channel 06 : 3[e2000] -> 9[e7000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,10]:dgx2h0194:41896:42859 [0] NCCL INFO Channel 05 : 10[e0000] -> 3[e2000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,1]:dgx2h0194:41887:42486 [0] NCCL INFO Channel 00 : 1[39000] -> 13[34000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,3]:dgx2h0194:41889:42722 [0] NCCL INFO Channel 07 : 3[e2000] -> 9[e7000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,10]:dgx2h0194:41896:42859 [0] NCCL INFO Channel 06 : 10[e0000] -> 3[e2000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,1]:dgx2h0194:41887:42486 [0] NCCL INFO Channel 01 : 1[39000] -> 13[34000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,3]:dgx2h0194:41889:42722 [0] NCCL INFO Channel 08 : 3[e2000] -> 9[e7000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,10]:dgx2h0194:41896:42859 [0] NCCL INFO Channel 07 : 10[e0000] -> 3[e2000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,6]:dgx2h0194:41892:42733 [0] NCCL INFO Connected all rings\n", - "Mon Sep 6 11:45:59 2021[1,1]:dgx2h0194:41887:42486 [0] NCCL INFO Channel 02 : 1[39000] -> 13[34000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,3]:dgx2h0194:41889:42722 [0] NCCL INFO Channel 09 : 3[e2000] -> 9[e7000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,10]:dgx2h0194:41896:42859 [0] NCCL INFO Channel 08 : 10[e0000] -> 3[e2000] via P2P/IPC\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:45:59 2021[1,6]:dgx2h0194:41892:42733 [0] NCCL INFO Channel 00 : 6[b7000] -> 8[be000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,1]:dgx2h0194:41887:42486 [0] NCCL INFO Channel 03 : 1[39000] -> 13[34000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,3]:dgx2h0194:41889:42722 [0] NCCL INFO Channel 10 : 3[e2000] -> 9[e7000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,10]:dgx2h0194:41896:42859 [0] NCCL INFO Channel 09 : 10[e0000] -> 3[e2000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,6]:dgx2h0194:41892:42733 [0] NCCL INFO Channel 01 : 6[b7000] -> 8[be000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,1]:dgx2h0194:41887:42486 [0] NCCL INFO Channel 04 : 1[39000] -> 13[34000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,3]:dgx2h0194:41889:42722 [0] NCCL INFO Channel 11 : 3[e2000] -> 9[e7000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,10]:dgx2h0194:41896:42859 [0] NCCL INFO Channel 10 : 10[e0000] -> 3[e2000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,6]:dgx2h0194:41892:42733 [0] NCCL INFO Channel 02 : 6[b7000] -> 8[be000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,1]:dgx2h0194:41887:42486 [0] NCCL INFO Channel 05 : 1[39000] -> 13[34000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,10]:dgx2h0194:41896:42859 [0] NCCL INFO Channel 11 : 10[e0000] -> 3[e2000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,6]:dgx2h0194:41892:42733 [0] NCCL INFO Channel 03 : 6[b7000] -> 8[be000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,1]:dgx2h0194:41887:42486 [0] NCCL INFO Channel 06 : 1[39000] -> 13[34000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,12]:dgx2h0194:41898:42413 [0] NCCL INFO Channel 00 : 12[57000] -> 5[59000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,6]:dgx2h0194:41892:42733 [0] NCCL INFO Channel 04 : 6[b7000] -> 8[be000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,1]:dgx2h0194:41887:42486 [0] NCCL INFO Channel 07 : 1[39000] -> 13[34000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,6]:dgx2h0194:41892:42733 [0] NCCL INFO Channel 05 : 6[b7000] -> 8[be000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,12]:dgx2h0194:41898:42413 [0] NCCL INFO Channel 01 : 12[57000] -> 5[59000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,1]:dgx2h0194:41887:42486 [0] NCCL INFO Channel 08 : 1[39000] -> 13[34000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,12]:dgx2h0194:41898:42413 [0] NCCL INFO Channel 02 : 12[57000] -> 5[59000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,6]:dgx2h0194:41892:42733 [0] NCCL INFO Channel 06 : 6[b7000] -> 8[be000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,1]:dgx2h0194:41887:42486 [0] NCCL INFO Channel 09 : 1[39000] -> 13[34000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,12]:dgx2h0194:41898:42413 [0] NCCL INFO Channel 03 : 12[57000] -> 5[59000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,6]:dgx2h0194:41892:42733 [0] NCCL INFO Channel 07 : 6[b7000] -> 8[be000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,1]:dgx2h0194:41887:42486 [0] NCCL INFO Channel 10 : 1[39000] -> 13[34000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,6]:dgx2h0194:41892:42733 [0] NCCL INFO Channel 08 : 6[b7000] -> 8[be000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,12]:dgx2h0194:41898:42413 [0] NCCL INFO Channel 04 : 12[57000] -> 5[59000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,1]:dgx2h0194:41887:42486 [0] NCCL INFO Channel 11 : 1[39000] -> 13[34000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,6]:dgx2h0194:41892:42733 [0] NCCL INFO Channel 09 : 6[b7000] -> 8[be000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,12]:dgx2h0194:41898:42413 [0] NCCL INFO Channel 05 : 12[57000] -> 5[59000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,2]:dgx2h0194:41888:42361 [0] NCCL INFO Channel 00 : 2[e5000] -> 11[5e000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,9]:dgx2h0194:41895:42860 [0] NCCL INFO Channel 00 : 9[e7000] -> 2[e5000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,6]:dgx2h0194:41892:42733 [0] NCCL INFO Channel 10 : 6[b7000] -> 8[be000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,2]:dgx2h0194:41888:42361 [0] NCCL INFO Channel 01 : 2[e5000] -> 11[5e000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,12]:dgx2h0194:41898:42413 [0] NCCL INFO Channel 06 : 12[57000] -> 5[59000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,9]:dgx2h0194:41895:42860 [0] NCCL INFO Channel 01 : 9[e7000] -> 2[e5000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,12]:dgx2h0194:41898:42413 [0] NCCL INFO Channel 07 : 12[57000] -> 5[59000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,2]:dgx2h0194:41888:42361 [0] NCCL INFO Channel 02 : 2[e5000] -> 11[5e000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,6]:dgx2h0194:41892:42733 [0] NCCL INFO Channel 11 : 6[b7000] -> 8[be000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,9]:dgx2h0194:41895:42860 [0] NCCL INFO Channel 02 : 9[e7000] -> 2[e5000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,2]:dgx2h0194:41888:42361 [0] NCCL INFO Channel 03 : 2[e5000] -> 11[5e000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,12]:dgx2h0194:41898:42413 [0] NCCL INFO Channel 08 : 12[57000] -> 5[59000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,9]:dgx2h0194:41895:42860 [0] NCCL INFO Channel 03 : 9[e7000] -> 2[e5000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,12]:dgx2h0194:41898:42413 [0] NCCL INFO Channel 09 : 12[57000] -> 5[59000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,2]:dgx2h0194:41888:42361 [0] NCCL INFO Channel 04 : 2[e5000] -> 11[5e000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,12]:dgx2h0194:41898:42413 [0] NCCL INFO Channel 10 : 12[57000] -> 5[59000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,2]:dgx2h0194:41888:42361 [0] NCCL INFO Channel 05 : 2[e5000] -> 11[5e000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,9]:dgx2h0194:41895:42860 [0] NCCL INFO Channel 04 : 9[e7000] -> 2[e5000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,12]:dgx2h0194:41898:42413 [0] NCCL INFO Channel 11 : 12[57000] -> 5[59000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,2]:dgx2h0194:41888:42361 [0] NCCL INFO Channel 06 : 2[e5000] -> 11[5e000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,9]:dgx2h0194:41895:42860 [0] NCCL INFO Channel 05 : 9[e7000] -> 2[e5000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,14]:dgx2h0194:41900:42734 [0] NCCL INFO Connected all trees\n", - "Mon Sep 6 11:45:59 2021[1,14]:dgx2h0194:41900:42734 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 8/8/512\n", - "Mon Sep 6 11:45:59 2021[1,1]:dgx2h0194:41887:42486 [0] NCCL INFO Connected all trees\n", - "Mon Sep 6 11:45:59 2021[1,1]:dgx2h0194:41887:42486 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 8/8/512\n", - "Mon Sep 6 11:45:59 2021[1,2]:dgx2h0194:41888:42361 [0] NCCL INFO Channel 07 : 2[e5000] -> 11[5e000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,13]:dgx2h0194:41899:42721 [0] NCCL INFO Connected all trees\n", - "Mon Sep 6 11:45:59 2021[1,13]:dgx2h0194:41899:42721 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 8/8/512\n", - "Mon Sep 6 11:45:59 2021[1,9]:dgx2h0194:41895:42860 [0] NCCL INFO Channel 06 : 9[e7000] -> 2[e5000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,2]:dgx2h0194:41888:42361 [0] NCCL INFO Channel 08 : 2[e5000] -> 11[5e000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,9]:dgx2h0194:41895:42860 [0] NCCL INFO Channel 07 : 9[e7000] -> 2[e5000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,14]:dgx2h0194:41900:42734 [0] NCCL INFO 12 coll channels, 16 p2p channels, 16 p2p channels per peer\n", - "Mon Sep 6 11:45:59 2021[1,1]:dgx2h0194:41887:42486 [0] NCCL INFO 12 coll channels, 16 p2p channels, 16 p2p channels per peer\n", - "Mon Sep 6 11:45:59 2021[1,13]:dgx2h0194:41899:42721 [0] NCCL INFO 12 coll channels, 16 p2p channels, 16 p2p channels per peer\n", - "Mon Sep 6 11:45:59 2021[1,2]:dgx2h0194:41888:42361 [0] NCCL INFO Channel 09 : 2[e5000] -> 11[5e000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,9]:dgx2h0194:41895:42860 [0] NCCL INFO Channel 08 : 9[e7000] -> 2[e5000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,2]:dgx2h0194:41888:42361 [0] NCCL INFO Channel 10 : 2[e5000] -> 11[5e000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,9]:dgx2h0194:41895:42860 [0] NCCL INFO Channel 09 : 9[e7000] -> 2[e5000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,8]:dgx2h0194:41894:42489 [0] NCCL INFO Channel 00 : 8[be000] -> 4[bc000] via P2P/IPC\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:45:59 2021[1,2]:dgx2h0194:41888:42361 [0] NCCL INFO Channel 11 : 2[e5000] -> 11[5e000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,9]:dgx2h0194:41895:42860 [0] NCCL INFO Channel 10 : 9[e7000] -> 2[e5000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,8]:dgx2h0194:41894:42489 [0] NCCL INFO Channel 01 : 8[be000] -> 4[bc000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,9]:dgx2h0194:41895:42860 [0] NCCL INFO Channel 11 : 9[e7000] -> 2[e5000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,8]:dgx2h0194:41894:42489 [0] NCCL INFO Channel 02 : 8[be000] -> 4[bc000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,12]:dgx2h0194:41898:42413 [0] NCCL INFO Connected all trees\n", - "Mon Sep 6 11:45:59 2021[1,12]:dgx2h0194:41898:42413 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 8/8/512\n", - "Mon Sep 6 11:45:59 2021[1,8]:dgx2h0194:41894:42489 [0] NCCL INFO Channel 03 : 8[be000] -> 4[bc000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,8]:dgx2h0194:41894:42489 [0] NCCL INFO Channel 04 : 8[be000] -> 4[bc000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,12]:dgx2h0194:41898:42413 [0] NCCL INFO 12 coll channels, 16 p2p channels, 16 p2p channels per peer\n", - "Mon Sep 6 11:45:59 2021[1,8]:dgx2h0194:41894:42489 [0] NCCL INFO Channel 05 : 8[be000] -> 4[bc000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,8]:dgx2h0194:41894:42489 [0] NCCL INFO Channel 06 : 8[be000] -> 4[bc000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,8]:dgx2h0194:41894:42489 [0] NCCL INFO Channel 07 : 8[be000] -> 4[bc000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,8]:dgx2h0194:41894:42489 [0] NCCL INFO Channel 08 : 8[be000] -> 4[bc000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,3]:dgx2h0194:41889:42722 [0] NCCL INFO Connected all trees\n", - "Mon Sep 6 11:45:59 2021[1,3]:dgx2h0194:41889:42722 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 8/8/512\n", - "Mon Sep 6 11:45:59 2021[1,10]:dgx2h0194:41896:42859 [0] NCCL INFO Connected all trees\n", - "Mon Sep 6 11:45:59 2021[1,10]:dgx2h0194:41896:42859 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 8/8/512\n", - "Mon Sep 6 11:45:59 2021[1,11]:dgx2h0194:41897:42858 [0] NCCL INFO Channel 00 : 11[5e000] -> 7[5c000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,8]:dgx2h0194:41894:42489 [0] NCCL INFO Channel 09 : 8[be000] -> 4[bc000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,9]:dgx2h0194:41895:42860 [0] NCCL INFO Connected all trees\n", - "Mon Sep 6 11:45:59 2021[1,9]:dgx2h0194:41895:42860 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 8/8/512\n", - "Mon Sep 6 11:45:59 2021[1,11]:dgx2h0194:41897:42858 [0] NCCL INFO Channel 01 : 11[5e000] -> 7[5c000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,8]:dgx2h0194:41894:42489 [0] NCCL INFO Channel 10 : 8[be000] -> 4[bc000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,5]:dgx2h0194:41891:42723 [0] NCCL INFO Connected all trees\n", - "Mon Sep 6 11:45:59 2021[1,5]:dgx2h0194:41891:42723 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 8/8/512\n", - "Mon Sep 6 11:45:59 2021[1,3]:dgx2h0194:41889:42722 [0] NCCL INFO 12 coll channels, 16 p2p channels, 16 p2p channels per peer\n", - "Mon Sep 6 11:45:59 2021[1,10]:dgx2h0194:41896:42859 [0] NCCL INFO 12 coll channels, 16 p2p channels, 16 p2p channels per peer\n", - "Mon Sep 6 11:45:59 2021[1,9]:dgx2h0194:41895:42860 [0] NCCL INFO 12 coll channels, 16 p2p channels, 16 p2p channels per peer\n", - "Mon Sep 6 11:45:59 2021[1,11]:dgx2h0194:41897:42858 [0] NCCL INFO Channel 02 : 11[5e000] -> 7[5c000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,8]:dgx2h0194:41894:42489 [0] NCCL INFO Channel 11 : 8[be000] -> 4[bc000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,5]:dgx2h0194:41891:42723 [0] NCCL INFO 12 coll channels, 16 p2p channels, 16 p2p channels per peer\n", - "Mon Sep 6 11:45:59 2021[1,11]:dgx2h0194:41897:42858 [0] NCCL INFO Channel 03 : 11[5e000] -> 7[5c000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,11]:dgx2h0194:41897:42858 [0] NCCL INFO Channel 04 : 11[5e000] -> 7[5c000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,11]:dgx2h0194:41897:42858 [0] NCCL INFO Channel 05 : 11[5e000] -> 7[5c000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,11]:dgx2h0194:41897:42858 [0] NCCL INFO Channel 06 : 11[5e000] -> 7[5c000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,11]:dgx2h0194:41897:42858 [0] NCCL INFO Channel 07 : 11[5e000] -> 7[5c000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,2]:dgx2h0194:41888:42361 [0] NCCL INFO Connected all trees\n", - "Mon Sep 6 11:45:59 2021[1,2]:dgx2h0194:41888:42361 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 8/8/512\n", - "Mon Sep 6 11:45:59 2021[1,11]:dgx2h0194:41897:42858 [0] NCCL INFO Channel 08 : 11[5e000] -> 7[5c000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,11]:dgx2h0194:41897:42858 [0] NCCL INFO Channel 09 : 11[5e000] -> 7[5c000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,2]:dgx2h0194:41888:42361 [0] NCCL INFO 12 coll channels, 16 p2p channels, 16 p2p channels per peer\n", - "Mon Sep 6 11:45:59 2021[1,6]:dgx2h0194:41892:42733 [0] NCCL INFO Connected all trees\n", - "Mon Sep 6 11:45:59 2021[1,6]:dgx2h0194:41892:42733 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 8/8/512\n", - "Mon Sep 6 11:45:59 2021[1,8]:dgx2h0194:41894:42489 [0] NCCL INFO Connected all trees\n", - "Mon Sep 6 11:45:59 2021[1,8]:dgx2h0194:41894:42489 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 8/8/512\n", - "Mon Sep 6 11:45:59 2021[1,4]:dgx2h0194:41890:42739 [0] NCCL INFO Connected all trees\n", - "Mon Sep 6 11:45:59 2021[1,4]:dgx2h0194:41890:42739 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 8/8/512\n", - "Mon Sep 6 11:45:59 2021[1,15]:dgx2h0194:41901:42857 [0] NCCL INFO Connected all trees\n", - "Mon Sep 6 11:45:59 2021[1,15]:dgx2h0194:41901:42857 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 8/8/512\n", - "Mon Sep 6 11:45:59 2021[1,11]:dgx2h0194:41897:42858 [0] NCCL INFO Channel 10 : 11[5e000] -> 7[5c000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,11]:dgx2h0194:41897:42858 [0] NCCL INFO Channel 11 : 11[5e000] -> 7[5c000] via P2P/IPC\n", - "Mon Sep 6 11:45:59 2021[1,6]:dgx2h0194:41892:42733 [0] NCCL INFO 12 coll channels, 16 p2p channels, 16 p2p channels per peer\n", - "Mon Sep 6 11:45:59 2021[1,8]:dgx2h0194:41894:42489 [0] NCCL INFO 12 coll channels, 16 p2p channels, 16 p2p channels per peer\n", - "Mon Sep 6 11:45:59 2021[1,4]:dgx2h0194:41890:42739 [0] NCCL INFO 12 coll channels, 16 p2p channels, 16 p2p channels per peer\n", - "Mon Sep 6 11:45:59 2021[1,15]:dgx2h0194:41901:42857 [0] NCCL INFO 12 coll channels, 16 p2p channels, 16 p2p channels per peer\n", - "Mon Sep 6 11:45:59 2021[1,11]:dgx2h0194:41897:42858 [0] NCCL INFO Connected all trees\n", - "Mon Sep 6 11:45:59 2021[1,11]:dgx2h0194:41897:42858 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 8/8/512\n", - "Mon Sep 6 11:45:59 2021[1,7]:dgx2h0194:41893:42743 [0] NCCL INFO Connected all trees\n", - "Mon Sep 6 11:45:59 2021[1,7]:dgx2h0194:41893:42743 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 8/8/512\n", - "Mon Sep 6 11:45:59 2021[1,11]:dgx2h0194:41897:42858 [0] NCCL INFO 12 coll channels, 16 p2p channels, 16 p2p channels per peer\n", - "Mon Sep 6 11:45:59 2021[1,7]:dgx2h0194:41893:42743 [0] NCCL INFO 12 coll channels, 16 p2p channels, 16 p2p channels per peer\n", - "Mon Sep 6 11:45:59 2021[1,7]:dgx2h0194:41893:42743 [0] NCCL INFO comm 0x7fdcb862ad60 rank 7 nranks 16 cudaDev 0 busId 5c000 - Init COMPLETE\n", - "Mon Sep 6 11:45:59 2021[1,11]:dgx2h0194:41897:42858 [0] NCCL INFO comm 0x7f2914629ae0 rank 11 nranks 16 cudaDev 0 busId 5e000 - Init COMPLETE\n", - "Mon Sep 6 11:45:59 2021[1,15]:dgx2h0194:41901:42857 [0] NCCL INFO comm 0x7fe41462a050 rank 15 nranks 16 cudaDev 0 busId b9000 - Init COMPLETE\n", - "Mon Sep 6 11:45:59 2021[1,13]:dgx2h0194:41899:42721 [0] NCCL INFO comm 0x7f3cf4629f50 rank 13 nranks 16 cudaDev 0 busId 34000 - Init COMPLETE\n", - "Mon Sep 6 11:45:59 2021[1,12]:dgx2h0194:41898:42413 [0] NCCL INFO comm 0x7fcea062aa60 rank 12 nranks 16 cudaDev 0 busId 57000 - Init COMPLETE\n", - "Mon Sep 6 11:45:59 2021[1,4]:dgx2h0194:41890:42739 [0] NCCL INFO comm 0x7f2bb862ae60 rank 4 nranks 16 cudaDev 0 busId bc000 - Init COMPLETE\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:45:59 2021[1,1]:dgx2h0194:41887:42486 [0] NCCL INFO comm 0x7fda3462aef0 rank 1 nranks 16 cudaDev 0 busId 39000 - Init COMPLETE\n", - "Mon Sep 6 11:45:59 2021[1,6]:dgx2h0194:41892:42733 [0] NCCL INFO comm 0x7f257462aab0 rank 6 nranks 16 cudaDev 0 busId b7000 - Init COMPLETE\n", - "Mon Sep 6 11:45:59 2021[1,2]:dgx2h0194:41888:42361 [0] NCCL INFO comm 0x7f54e462ac10 rank 2 nranks 16 cudaDev 0 busId e5000 - Init COMPLETE\n", - "Mon Sep 6 11:45:59 2021[1,8]:dgx2h0194:41894:42489 [0] NCCL INFO comm 0x7fd8c862b120 rank 8 nranks 16 cudaDev 0 busId be000 - Init COMPLETE\n", - "Mon Sep 6 11:45:59 2021[1,3]:dgx2h0194:41889:42722 [0] NCCL INFO comm 0x7f33b062a600 rank 3 nranks 16 cudaDev 0 busId e2000 - Init COMPLETE\n", - "Mon Sep 6 11:45:59 2021[1,5]:dgx2h0194:41891:42723 [0] NCCL INFO comm 0x7fcf1462aa00 rank 5 nranks 16 cudaDev 0 busId 59000 - Init COMPLETE\n", - "Mon Sep 6 11:45:59 2021[1,9]:dgx2h0194:41895:42860 [0] NCCL INFO comm 0x7fb7d062a3c0 rank 9 nranks 16 cudaDev 0 busId e7000 - Init COMPLETE\n", - "Mon Sep 6 11:45:59 2021[1,0]:dgx2h0194:41886:42730 [0] NCCL INFO comm 0x7f1c8068aa60 rank 0 nranks 16 cudaDev 0 busId 36000 - Init COMPLETE\n", - "Mon Sep 6 11:45:59 2021[1,14]:dgx2h0194:41900:42734 [0] NCCL INFO comm 0x7f7db062a080 rank 14 nranks 16 cudaDev 0 busId 3b000 - Init COMPLETE\n", - "Mon Sep 6 11:45:59 2021[1,10]:dgx2h0194:41896:42859 [0] NCCL INFO comm 0x7f18ac629ef0 rank 10 nranks 16 cudaDev 0 busId e0000 - Init COMPLETE\n", - "Mon Sep 6 11:45:59 2021[1,0]:dgx2h0194:41886:42730 [0] NCCL INFO Launch mode Parallel\n", - "Mon Sep 6 11:45:59 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:46:11 2021[1,11]:2021-09-06 11:46:11.401104: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 11:46:11 2021[1,11]:2021-09-06 11:46:11.401164: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 11:46:11 2021[1,3]:2021-09-06 11:46:11.440290: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 11:46:11 2021[1,3]:2021-09-06 11:46:11.440344: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 11:46:11 2021[1,9]:2021-09-06 11:46:11.442416: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 11:46:11 2021[1,9]:2021-09-06 11:46:11.442453: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 11:46:11 2021[1,5]:2021-09-06 11:46:11.445379: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 11:46:11 2021[1,5]:2021-09-06 11:46:11.445409: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 11:46:11 2021[1,8]:2021-09-06 11:46:11.461500: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 11:46:11 2021[1,8]:2021-09-06 11:46:11.461545: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 11:46:11 2021[1,0]:2021-09-06 11:46:11.467532: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 11:46:11 2021[1,0]:2021-09-06 11:46:11.467566: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 11:46:11 2021[1,14]:2021-09-06 11:46:11.467699: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 11:46:11 2021[1,14]:2021-09-06 11:46:11.467725: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 11:46:11 2021[1,7]:2021-09-06 11:46:11.467927: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 11:46:11 2021[1,7]:2021-09-06 11:46:11.467954: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 11:46:11 2021[1,15]:2021-09-06 11:46:11.478742: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 11:46:11 2021[1,15]:2021-09-06 11:46:11.478775: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 11:46:11 2021[1,2]:2021-09-06 11:46:11.480342: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 11:46:11 2021[1,2]:2021-09-06 11:46:11.480375: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 11:46:11 2021[1,4]:2021-09-06 11:46:11.493077: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 11:46:11 2021[1,4]:2021-09-06 11:46:11.493108: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 11:46:11 2021[1,12]:2021-09-06 11:46:11.514149: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 11:46:11 2021[1,12]:2021-09-06 11:46:11.514199: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 11:46:11 2021[1,1]:2021-09-06 11:46:11.517690: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 11:46:11 2021[1,1]:2021-09-06 11:46:11.517726: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 11:46:11 2021[1,13]:2021-09-06 11:46:11.526581: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 11:46:11 2021[1,13]:2021-09-06 11:46:11.526613: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 11:46:11 2021[1,6]:2021-09-06 11:46:11.537704: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 11:46:11 2021[1,6]:2021-09-06 11:46:11.537739: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 11:46:11 2021[1,10]:2021-09-06 11:46:11.555526: I tensorflow/core/profiler/lib/profiler_session.cc:131] Profiler session initializing.\n", - "Mon Sep 6 11:46:11 2021[1,10]:2021-09-06 11:46:11.555565: I tensorflow/core/profiler/lib/profiler_session.cc:146] Profiler session started.\n", - "Mon Sep 6 11:46:12 2021[1,13]:2021-09-06 11:46:12.486392: I tensorflow/core/profiler/lib/profiler_session.cc:66] Profiler session collecting data.\n", - "Mon Sep 6 11:46:12 2021[1,14]:2021-09-06 11:46:12.486420: I tensorflow/core/profiler/lib/profiler_session.cc:66] Profiler session collecting data.\n", - "Mon Sep 6 11:46:12 2021[1,2]:2021-09-06 11:46:12.486804: I tensorflow/core/profiler/lib/profiler_session.cc:66] Profiler session collecting data.\n", - "Mon Sep 6 11:46:12 2021[1,6]:2021-09-06 11:46:12.486894: I tensorflow/core/profiler/lib/profiler_session.cc:66] Profiler session collecting data.\n", - "Mon Sep 6 11:46:12 2021[1,11]:2021-09-06 11:46:12.487354: I tensorflow/core/profiler/lib/profiler_session.cc:66] Profiler session collecting data.\n", - "Mon Sep 6 11:46:12 2021[1,6]:2021-09-06 11:46:12.489094: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 11:46:12 2021[1,13]:2021-09-06 11:46:12.489218: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 11:46:12 2021[1,2]:2021-09-06 11:46:12.489982: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 11:46:12 2021[1,11]:2021-09-06 11:46:12.490307: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 11:46:12 2021[1,14]:2021-09-06 11:46:12.490535: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:45:59 2021[1,0]: 1/4000 [..............................]Mon Sep 6 11:45:59 2021[1,0]: - ETA: 34:42:44 - loss: 1.3348 - auc: 0.5421Mon Sep 6 11:46:12 2021[1,0]:\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\bMon Sep 6 11:46:12 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:46:12 2021[1,5]:2021-09-06 11:46:12.494098: I tensorflow/core/profiler/lib/profiler_session.cc:66] Profiler session collecting data.\n", - "Mon Sep 6 11:46:12 2021[1,9]:2021-09-06 11:46:12.494182: I tensorflow/core/profiler/lib/profiler_session.cc:66] Profiler session collecting data.\n", - "Mon Sep 6 11:46:12 2021[1,12]:2021-09-06 11:46:12.494154: I tensorflow/core/profiler/lib/profiler_session.cc:66] Profiler session collecting data.\n", - "Mon Sep 6 11:46:12 2021[1,15]:2021-09-06 11:46:12.494339: I tensorflow/core/profiler/lib/profiler_session.cc:66] Profiler session collecting data.\n", - "Mon Sep 6 11:46:12 2021[1,8]:2021-09-06 11:46:12.494413: I tensorflow/core/profiler/lib/profiler_session.cc:66] Profiler session collecting data.\n", - "Mon Sep 6 11:46:12 2021[1,3]:2021-09-06 11:46:12.494461: I tensorflow/core/profiler/lib/profiler_session.cc:66] Profiler session collecting data.\n", - "Mon Sep 6 11:46:12 2021[1,1]:2021-09-06 11:46:12.494477: I tensorflow/core/profiler/lib/profiler_session.cc:66] Profiler session collecting data.\n", - "Mon Sep 6 11:46:12 2021[1,4]:2021-09-06 11:46:12.494922: I tensorflow/core/profiler/lib/profiler_session.cc:66] Profiler session collecting data.\n", - "Mon Sep 6 11:46:12 2021[1,10]:2021-09-06 11:46:12.495422: I tensorflow/core/profiler/lib/profiler_session.cc:66] Profiler session collecting data.\n", - "Mon Sep 6 11:46:12 2021[1,12]:2021-09-06 11:46:12.496543: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 11:46:12 2021[1,9]:2021-09-06 11:46:12.496573: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 11:46:12 2021[1,0]:2021-09-06 11:46:12.496713: I tensorflow/core/profiler/lib/profiler_session.cc:66] Profiler session collecting data.\n", - "Mon Sep 6 11:46:12 2021[1,5]:2021-09-06 11:46:12.496875: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 11:46:12 2021[1,7]:2021-09-06 11:46:12.497310: I tensorflow/core/profiler/lib/profiler_session.cc:66] Profiler session collecting data.\n", - "Mon Sep 6 11:46:12 2021[1,8]:2021-09-06 11:46:12.497408: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 11:46:12 2021[1,1]:2021-09-06 11:46:12.497466: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 11:46:12 2021[1,15]:2021-09-06 11:46:12.497573: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 11:46:12 2021[1,10]:2021-09-06 11:46:12.497718: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 11:46:12 2021[1,4]:2021-09-06 11:46:12.497957: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 11:46:12 2021[1,3]:2021-09-06 11:46:12.498785: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 11:46:12 2021[1,0]:2021-09-06 11:46:12.499468: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 11:46:12 2021[1,7]:2021-09-06 11:46:12.499616: I tensorflow/core/profiler/internal/gpu/cupti_tracer.cc:1748] CUPTI activity buffer flushed\n", - "Mon Sep 6 11:46:12 2021[1,10]:2021-09-06 11:46:12.689220: I tensorflow/core/profiler/internal/gpu/cupti_collector.cc:673] GpuTracer has collected 2615 callback api events and 2562 activity events. \n", - "Mon Sep 6 11:46:12 2021[1,7]:2021-09-06 11:46:12.738674: I tensorflow/core/profiler/internal/gpu/cupti_collector.cc:673] GpuTracer has collected 2612 callback api events and 2559 activity events. \n", - "Mon Sep 6 11:46:12 2021[1,10]:2021-09-06 11:46:12.739311: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 11:46:12 2021[1,2]:2021-09-06 11:46:12.772937: I tensorflow/core/profiler/internal/gpu/cupti_collector.cc:673] GpuTracer has collected 2614 callback api events and 2561 activity events. \n", - "Mon Sep 6 11:46:12 2021[1,13]:2021-09-06 11:46:12.775817: I tensorflow/core/profiler/internal/gpu/cupti_collector.cc:673] GpuTracer has collected 2615 callback api events and 2562 activity events. \n", - "Mon Sep 6 11:46:12 2021[1,0]:2021-09-06 11:46:12.788372: I tensorflow/core/profiler/internal/gpu/cupti_collector.cc:673] GpuTracer has collected 2614 callback api events and 2561 activity events. \n", - "Mon Sep 6 11:46:12 2021[1,11]:2021-09-06 11:46:12.793984: I tensorflow/core/profiler/internal/gpu/cupti_collector.cc:673] GpuTracer has collected 2615 callback api events and 2562 activity events. \n", - "Mon Sep 6 11:46:12 2021[1,14]:2021-09-06 11:46:12.797338: I tensorflow/core/profiler/internal/gpu/cupti_collector.cc:673] GpuTracer has collected 2615 callback api events and 2562 activity events. \n", - "Mon Sep 6 11:46:12 2021[1,12]:2021-09-06 11:46:12.799541: I tensorflow/core/profiler/internal/gpu/cupti_collector.cc:673] GpuTracer has collected 2614 callback api events and 2561 activity events. \n", - "Mon Sep 6 11:46:12 2021[1,3]:2021-09-06 11:46:12.804509: I tensorflow/core/profiler/internal/gpu/cupti_collector.cc:673] GpuTracer has collected 2615 callback api events and 2562 activity events. \n", - "Mon Sep 6 11:46:12 2021[1,9]:2021-09-06 11:46:12.809215: I tensorflow/core/profiler/internal/gpu/cupti_collector.cc:673] GpuTracer has collected 2614 callback api events and 2561 activity events. \n", - "Mon Sep 6 11:46:12 2021[1,6]:2021-09-06 11:46:12.817765: I tensorflow/core/profiler/internal/gpu/cupti_collector.cc:673] GpuTracer has collected 2615 callback api events and 2562 activity events. \n", - "Mon Sep 6 11:46:12 2021[1,7]:2021-09-06 11:46:12.825298: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 11:46:12 2021[1,4]:2021-09-06 11:46:12.830887: I tensorflow/core/profiler/internal/gpu/cupti_collector.cc:673] GpuTracer has collected 2612 callback api events and 2559 activity events. \n", - "Mon Sep 6 11:46:12 2021[1,15]:2021-09-06 11:46:12.831932: I tensorflow/core/profiler/internal/gpu/cupti_collector.cc:673] GpuTracer has collected 2612 callback api events and 2559 activity events. \n", - "Mon Sep 6 11:46:12 2021[1,5]:2021-09-06 11:46:12.836523: I tensorflow/core/profiler/internal/gpu/cupti_collector.cc:673] GpuTracer has collected 2615 callback api events and 2562 activity events. \n", - "Mon Sep 6 11:46:12 2021[1,2]:2021-09-06 11:46:12.838553: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 11:46:12 2021[1,13]:2021-09-06 11:46:12.839676: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 11:46:12 2021[1,1]:2021-09-06 11:46:12.840641: I tensorflow/core/profiler/internal/gpu/cupti_collector.cc:673] GpuTracer has collected 2615 callback api events and 2562 activity events. \n", - "Mon Sep 6 11:46:12 2021[1,8]:2021-09-06 11:46:12.847102: I tensorflow/core/profiler/internal/gpu/cupti_collector.cc:673] GpuTracer has collected 2615 callback api events and 2562 activity events. \n", - "Mon Sep 6 11:46:12 2021[1,0]:2021-09-06 11:46:12.851656: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 11:46:12 2021[1,12]:2021-09-06 11:46:12.862278: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 11:46:12 2021[1,11]:2021-09-06 11:46:12.862736: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 11:46:12 2021[1,14]:2021-09-06 11:46:12.865552: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 11:46:12 2021[1,9]:2021-09-06 11:46:12.874796: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 11:46:12 2021[1,3]:2021-09-06 11:46:12.883307: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:46:12 2021[1,4]:2021-09-06 11:46:12.908955: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 11:46:12 2021[1,5]:2021-09-06 11:46:12.909348: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 11:46:12 2021[1,15]:2021-09-06 11:46:12.911988: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 11:46:12 2021[1,6]:2021-09-06 11:46:12.913338: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 11:46:12 2021[1,8]:2021-09-06 11:46:12.919742: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 11:46:12 2021[1,1]:2021-09-06 11:46:12.921926: I tensorflow/core/profiler/lib/profiler_session.cc:164] Profiler session tear down.\n", - "Mon Sep 6 11:46:12 2021[1,10]:2021-09-06 11:46:12.971783: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:12 2021[1,10]:\n", - "Mon Sep 6 11:46:13 2021[1,10]:2021-09-06 11:46:13.126279: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for trace.json.gz to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.trace.json.gz\n", - "Mon Sep 6 11:46:13 2021[1,7]:2021-09-06 11:46:13.220381: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:13 2021[1,7]:\n", - "Mon Sep 6 11:46:13 2021[1,10]:2021-09-06 11:46:13.236756: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:13 2021[1,10]:\n", - "Mon Sep 6 11:46:13 2021[1,10]:2021-09-06 11:46:13.244220: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for memory_profile.json.gz to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.memory_profile.json.gz\n", - "Mon Sep 6 11:46:13 2021[1,0]:2021-09-06 11:46:13.247347: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:13 2021[1,0]:\n", - "Mon Sep 6 11:46:13 2021[1,10]:2021-09-06 11:46:13.248759: I tensorflow/core/profiler/rpc/client/capture_profile.cc:251] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:13 2021[1,10]:Dumped tool data for xplane.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.xplane.pb\n", - "Mon Sep 6 11:46:13 2021[1,10]:Dumped tool data for overview_page.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.overview_page.pb\n", - "Mon Sep 6 11:46:13 2021[1,10]:Dumped tool data for input_pipeline.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.input_pipeline.pb\n", - "Mon Sep 6 11:46:13 2021[1,10]:Dumped tool data for tensorflow_stats.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.tensorflow_stats.pb\n", - "Mon Sep 6 11:46:13 2021[1,10]:Dumped tool data for kernel_stats.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.kernel_stats.pb\n", - "Mon Sep 6 11:46:13 2021[1,10]:\n", - "Mon Sep 6 11:46:13 2021[1,12]:2021-09-06 11:46:13.282450: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:13 2021[1,12]:\n", - "Mon Sep 6 11:46:13 2021[1,9]:2021-09-06 11:46:13.305546: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:13 2021[1,9]:\n", - "Mon Sep 6 11:46:13 2021[1,13]:2021-09-06 11:46:13.314358: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:13 2021[1,13]:\n", - "Mon Sep 6 11:46:13 2021[1,14]:2021-09-06 11:46:13.347053: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:13 2021[1,14]:\n", - "Mon Sep 6 11:46:13 2021[1,11]:2021-09-06 11:46:13.389071: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:13 2021[1,11]:\n", - "Mon Sep 6 11:46:13 2021[1,8]:2021-09-06 11:46:13.417449: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:13 2021[1,8]:\n", - "Mon Sep 6 11:46:13 2021[1,15]:2021-09-06 11:46:13.440662: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:13 2021[1,15]:\n", - "Mon Sep 6 11:46:13 2021[1,7]:2021-09-06 11:46:13.445472: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for trace.json.gz to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.trace.json.gz\n", - "Mon Sep 6 11:46:13 2021[1,5]:2021-09-06 11:46:13.464452: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:13 2021[1,5]:\n", - "Mon Sep 6 11:46:13 2021[1,4]:2021-09-06 11:46:13.505475: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:13 2021[1,4]:\n", - "Mon Sep 6 11:46:13 2021[1,1]:2021-09-06 11:46:13.538948: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:13 2021[1,1]:\n", - "Mon Sep 6 11:46:13 2021[1,6]:2021-09-06 11:46:13.539917: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:13 2021[1,6]:\n", - "Mon Sep 6 11:46:13 2021[1,7]:2021-09-06 11:46:13.575977: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:13 2021[1,7]:\n", - "Mon Sep 6 11:46:13 2021[1,7]:2021-09-06 11:46:13.583159: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for memory_profile.json.gz to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.memory_profile.json.gz\n", - "Mon Sep 6 11:46:13 2021[1,7]:2021-09-06 11:46:13.588595: I tensorflow/core/profiler/rpc/client/capture_profile.cc:251] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:13 2021[1,7]:Dumped tool data for xplane.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.xplane.pb\n", - "Mon Sep 6 11:46:13 2021[1,7]:Dumped tool data for overview_page.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.overview_page.pb\n", - "Mon Sep 6 11:46:13 2021[1,7]:Dumped tool data for input_pipeline.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.input_pipeline.pb\n", - "Mon Sep 6 11:46:13 2021[1,7]:Dumped tool data for tensorflow_stats.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.tensorflow_stats.pb\n", - "Mon Sep 6 11:46:13 2021[1,7]:Dumped tool data for kernel_stats.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.kernel_stats.pb\n", - "Mon Sep 6 11:46:13 2021[1,7]:\n", - "Mon Sep 6 11:46:13 2021[1,0]:2021-09-06 11:46:13.596438: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for trace.json.gz to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.trace.json.gz\n", - "Mon Sep 6 11:46:13 2021[1,12]:2021-09-06 11:46:13.663240: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for trace.json.gz to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.trace.json.gz\n", - "Mon Sep 6 11:46:13 2021[1,9]:2021-09-06 11:46:13.669575: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for trace.json.gz to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.trace.json.gz\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:46:13 2021[1,3]:2021-09-06 11:46:13.705164: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:13 2021[1,3]:\n", - "Mon Sep 6 11:46:13 2021[1,2]:2021-09-06 11:46:13.727985: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:13 2021[1,2]:\n", - "Mon Sep 6 11:46:13 2021[1,14]:2021-09-06 11:46:13.770514: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for trace.json.gz to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.trace.json.gz\n", - "Mon Sep 6 11:46:13 2021[1,0]:2021-09-06 11:46:13.770742: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:13 2021[1,0]:\n", - "Mon Sep 6 11:46:13 2021[1,0]:2021-09-06 11:46:13.777869: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for memory_profile.json.gz to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.memory_profile.json.gz\n", - "Mon Sep 6 11:46:13 2021[1,0]:2021-09-06 11:46:13.784817: I tensorflow/core/profiler/rpc/client/capture_profile.cc:251] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:13 2021[1,0]:Dumped tool data for xplane.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.xplane.pb\n", - "Mon Sep 6 11:46:13 2021[1,0]:Dumped tool data for overview_page.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.overview_page.pb\n", - "Mon Sep 6 11:46:13 2021[1,0]:Dumped tool data for input_pipeline.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.input_pipeline.pb\n", - "Mon Sep 6 11:46:13 2021[1,0]:Dumped tool data for tensorflow_stats.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.tensorflow_stats.pb\n", - "Mon Sep 6 11:46:13 2021[1,0]:Dumped tool data for kernel_stats.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.kernel_stats.pb\n", - "Mon Sep 6 11:46:13 2021[1,0]:\n", - "Mon Sep 6 11:46:13 2021[1,13]:2021-09-06 11:46:13.806429: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for trace.json.gz to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.trace.json.gz\n", - "Mon Sep 6 11:46:13 2021[1,12]:2021-09-06 11:46:13.836961: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:13 2021[1,12]:\n", - "Mon Sep 6 11:46:13 2021[1,12]:2021-09-06 11:46:13.844073: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for memory_profile.json.gz to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.memory_profile.json.gz\n", - "Mon Sep 6 11:46:13 2021[1,12]:2021-09-06 11:46:13.850941: I tensorflow/core/profiler/rpc/client/capture_profile.cc:251] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:13 2021[1,12]:Dumped tool data for xplane.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.xplane.pb\n", - "Mon Sep 6 11:46:13 2021[1,12]:Dumped tool data for overview_page.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.overview_page.pb\n", - "Mon Sep 6 11:46:13 2021[1,12]:Dumped tool data for input_pipeline.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.input_pipeline.pb\n", - "Mon Sep 6 11:46:13 2021[1,12]:Dumped tool data for tensorflow_stats.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.tensorflow_stats.pb\n", - "Mon Sep 6 11:46:13 2021[1,12]:Dumped tool data for kernel_stats.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.kernel_stats.pb\n", - "Mon Sep 6 11:46:13 2021[1,12]:\n", - "Mon Sep 6 11:46:13 2021[1,9]:2021-09-06 11:46:13.852005: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:13 2021[1,9]:\n", - "Mon Sep 6 11:46:13 2021[1,8]:2021-09-06 11:46:13.858089: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for trace.json.gz to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.trace.json.gz\n", - "Mon Sep 6 11:46:13 2021[1,9]:2021-09-06 11:46:13.858866: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for memory_profile.json.gz to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.memory_profile.json.gz\n", - "Mon Sep 6 11:46:13 2021[1,9]:2021-09-06 11:46:13.865756: I tensorflow/core/profiler/rpc/client/capture_profile.cc:251] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:13 2021[1,9]:Dumped tool data for xplane.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.xplane.pb\n", - "Mon Sep 6 11:46:13 2021[1,9]:Dumped tool data for overview_page.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.overview_page.pb\n", - "Mon Sep 6 11:46:13 2021[1,9]:Dumped tool data for input_pipeline.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.input_pipeline.pb\n", - "Mon Sep 6 11:46:13 2021[1,9]:Dumped tool data for tensorflow_stats.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.tensorflow_stats.pb\n", - "Mon Sep 6 11:46:13 2021[1,9]:Dumped tool data for kernel_stats.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.kernel_stats.pb\n", - "Mon Sep 6 11:46:13 2021[1,9]:\n", - "Mon Sep 6 11:46:13 2021[1,1]:2021-09-06 11:46:13.919252: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for trace.json.gz to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.trace.json.gz\n", - "Mon Sep 6 11:46:13 2021[1,5]:2021-09-06 11:46:13.921579: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for trace.json.gz to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.trace.json.gz\n", - "Mon Sep 6 11:46:13 2021[1,15]:2021-09-06 11:46:13.935380: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for trace.json.gz to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.trace.json.gz\n", - "Mon Sep 6 11:46:13 2021[1,11]:2021-09-06 11:46:13.953863: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for trace.json.gz to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.trace.json.gz\n", - "Mon Sep 6 11:46:13 2021[1,14]:2021-09-06 11:46:13.969342: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:13 2021[1,14]:\n", - "Mon Sep 6 11:46:13 2021[1,14]:2021-09-06 11:46:13.976218: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for memory_profile.json.gz to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.memory_profile.json.gz\n", - "Mon Sep 6 11:46:13 2021[1,14]:2021-09-06 11:46:13.983131: I tensorflow/core/profiler/rpc/client/capture_profile.cc:251] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:13 2021[1,14]:Dumped tool data for xplane.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.xplane.pb\n", - "Mon Sep 6 11:46:13 2021[1,14]:Dumped tool data for overview_page.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.overview_page.pb\n", - "Mon Sep 6 11:46:13 2021[1,14]:Dumped tool data for input_pipeline.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.input_pipeline.pb\n", - "Mon Sep 6 11:46:13 2021[1,14]:Dumped tool data for tensorflow_stats.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.tensorflow_stats.pb\n", - "Mon Sep 6 11:46:13 2021[1,14]:Dumped tool data for kernel_stats.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.kernel_stats.pb\n", - "Mon Sep 6 11:46:13 2021[1,14]:\n", - "Mon Sep 6 11:46:14 2021[1,13]:2021-09-06 11:46:14.025457: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:14 2021[1,13]:\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:46:14 2021[1,4]:2021-09-06 11:46:14.031628: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for trace.json.gz to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.trace.json.gz\n", - "Mon Sep 6 11:46:14 2021[1,13]:2021-09-06 11:46:14.035052: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for memory_profile.json.gz to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.memory_profile.json.gz\n", - "Mon Sep 6 11:46:14 2021[1,13]:2021-09-06 11:46:14.044101: I tensorflow/core/profiler/rpc/client/capture_profile.cc:251] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:14 2021[1,13]:Dumped tool data for xplane.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.xplane.pb\n", - "Mon Sep 6 11:46:14 2021[1,13]:Dumped tool data for overview_page.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.overview_page.pb\n", - "Mon Sep 6 11:46:14 2021[1,13]:Dumped tool data for input_pipeline.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.input_pipeline.pb\n", - "Mon Sep 6 11:46:14 2021[1,13]:Dumped tool data for tensorflow_stats.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.tensorflow_stats.pb\n", - "Mon Sep 6 11:46:14 2021[1,13]:Dumped tool data for kernel_stats.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.kernel_stats.pb\n", - "Mon Sep 6 11:46:14 2021[1,13]:\n", - "Mon Sep 6 11:46:14 2021[1,6]:2021-09-06 11:46:14.051427: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for trace.json.gz to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.trace.json.gz\n", - "Mon Sep 6 11:46:14 2021[1,8]:2021-09-06 11:46:14.066023: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:14 2021[1,8]:\n", - "Mon Sep 6 11:46:14 2021[1,8]:2021-09-06 11:46:14.073202: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for memory_profile.json.gz to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.memory_profile.json.gz\n", - "Mon Sep 6 11:46:14 2021[1,8]:2021-09-06 11:46:14.080723: I tensorflow/core/profiler/rpc/client/capture_profile.cc:251] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:14 2021[1,8]:Dumped tool data for xplane.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.xplane.pb\n", - "Mon Sep 6 11:46:14 2021[1,8]:Dumped tool data for overview_page.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.overview_page.pb\n", - "Mon Sep 6 11:46:14 2021[1,8]:Dumped tool data for input_pipeline.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.input_pipeline.pb\n", - "Mon Sep 6 11:46:14 2021[1,8]:Dumped tool data for tensorflow_stats.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.tensorflow_stats.pb\n", - "Mon Sep 6 11:46:14 2021[1,8]:Dumped tool data for kernel_stats.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.kernel_stats.pb\n", - "Mon Sep 6 11:46:14 2021[1,8]:\n", - "Mon Sep 6 11:46:14 2021[1,1]:2021-09-06 11:46:14.110787: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:14 2021[1,1]:\n", - "Mon Sep 6 11:46:14 2021[1,1]:2021-09-06 11:46:14.118317: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for memory_profile.json.gz to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.memory_profile.json.gz\n", - "Mon Sep 6 11:46:14 2021[1,1]:2021-09-06 11:46:14.125946: I tensorflow/core/profiler/rpc/client/capture_profile.cc:251] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:14 2021[1,1]:Dumped tool data for xplane.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.xplane.pb\n", - "Mon Sep 6 11:46:14 2021[1,1]:Dumped tool data for overview_page.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.overview_page.pb\n", - "Mon Sep 6 11:46:14 2021[1,1]:Dumped tool data for input_pipeline.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.input_pipeline.pb\n", - "Mon Sep 6 11:46:14 2021[1,1]:Dumped tool data for tensorflow_stats.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.tensorflow_stats.pb\n", - "Mon Sep 6 11:46:14 2021[1,1]:Dumped tool data for kernel_stats.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.kernel_stats.pb\n", - "Mon Sep 6 11:46:14 2021[1,1]:\n", - "Mon Sep 6 11:46:14 2021[1,5]:2021-09-06 11:46:14.131987: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:14 2021[1,5]:\n", - "Mon Sep 6 11:46:14 2021[1,5]:2021-09-06 11:46:14.138893: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for memory_profile.json.gz to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.memory_profile.json.gz\n", - "Mon Sep 6 11:46:14 2021[1,3]:2021-09-06 11:46:14.143929: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for trace.json.gz to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.trace.json.gz\n", - "Mon Sep 6 11:46:14 2021[1,5]:2021-09-06 11:46:14.146526: I tensorflow/core/profiler/rpc/client/capture_profile.cc:251] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:14 2021[1,5]:Dumped tool data for xplane.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.xplane.pb\n", - "Mon Sep 6 11:46:14 2021[1,5]:Dumped tool data for overview_page.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.overview_page.pb\n", - "Mon Sep 6 11:46:14 2021[1,5]:Dumped tool data for input_pipeline.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.input_pipeline.pb\n", - "Mon Sep 6 11:46:14 2021[1,5]:Dumped tool data for tensorflow_stats.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.tensorflow_stats.pb\n", - "Mon Sep 6 11:46:14 2021[1,5]:Dumped tool data for kernel_stats.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.kernel_stats.pb\n", - "Mon Sep 6 11:46:14 2021[1,5]:\n", - "Mon Sep 6 11:46:14 2021[1,15]:2021-09-06 11:46:14.170109: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:14 2021[1,15]:\n", - "Mon Sep 6 11:46:14 2021[1,11]:2021-09-06 11:46:14.176378: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:14 2021[1,11]:\n", - "Mon Sep 6 11:46:14 2021[1,15]:2021-09-06 11:46:14.177822: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for memory_profile.json.gz to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.memory_profile.json.gz\n", - "Mon Sep 6 11:46:14 2021[1,11]:2021-09-06 11:46:14.184029: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for memory_profile.json.gz to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.memory_profile.json.gz\n", - "Mon Sep 6 11:46:14 2021[1,15]:2021-09-06 11:46:14.186611: I tensorflow/core/profiler/rpc/client/capture_profile.cc:251] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:14 2021[1,15]:Dumped tool data for xplane.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.xplane.pb\n", - "Mon Sep 6 11:46:14 2021[1,15]:Dumped tool data for overview_page.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.overview_page.pb\n", - "Mon Sep 6 11:46:14 2021[1,15]:Dumped tool data for input_pipeline.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.input_pipeline.pb\n", - "Mon Sep 6 11:46:14 2021[1,15]:Dumped tool data for tensorflow_stats.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.tensorflow_stats.pb\n", - "Mon Sep 6 11:46:14 2021[1,15]:Dumped tool data for kernel_stats.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.kernel_stats.pb\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:46:14 2021[1,15]:\n", - "Mon Sep 6 11:46:14 2021[1,11]:2021-09-06 11:46:14.192490: I tensorflow/core/profiler/rpc/client/capture_profile.cc:251] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:14 2021[1,11]:Dumped tool data for xplane.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.xplane.pb\n", - "Mon Sep 6 11:46:14 2021[1,11]:Dumped tool data for overview_page.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.overview_page.pb\n", - "Mon Sep 6 11:46:14 2021[1,11]:Dumped tool data for input_pipeline.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.input_pipeline.pb\n", - "Mon Sep 6 11:46:14 2021[1,11]:Dumped tool data for tensorflow_stats.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.tensorflow_stats.pb\n", - "Mon Sep 6 11:46:14 2021[1,11]:Dumped tool data for kernel_stats.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.kernel_stats.pb\n", - "Mon Sep 6 11:46:14 2021[1,11]:\n", - "Mon Sep 6 11:46:14 2021[1,6]:2021-09-06 11:46:14.235507: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:14 2021[1,6]:\n", - "Mon Sep 6 11:46:14 2021[1,6]:2021-09-06 11:46:14.242545: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for memory_profile.json.gz to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.memory_profile.json.gz\n", - "Mon Sep 6 11:46:14 2021[1,2]:2021-09-06 11:46:14.242914: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for trace.json.gz to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.trace.json.gz\n", - "Mon Sep 6 11:46:14 2021[1,6]:2021-09-06 11:46:14.249229: I tensorflow/core/profiler/rpc/client/capture_profile.cc:251] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:14 2021[1,6]:Dumped tool data for xplane.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.xplane.pb\n", - "Mon Sep 6 11:46:14 2021[1,6]:Dumped tool data for overview_page.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.overview_page.pb\n", - "Mon Sep 6 11:46:14 2021[1,6]:Dumped tool data for input_pipeline.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.input_pipeline.pb\n", - "Mon Sep 6 11:46:14 2021[1,6]:Dumped tool data for tensorflow_stats.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.tensorflow_stats.pb\n", - "Mon Sep 6 11:46:14 2021[1,6]:Dumped tool data for kernel_stats.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.kernel_stats.pb\n", - "Mon Sep 6 11:46:14 2021[1,6]:\n", - "Mon Sep 6 11:46:14 2021[1,4]:2021-09-06 11:46:14.293566: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:14 2021[1,4]:\n", - "Mon Sep 6 11:46:14 2021[1,4]:2021-09-06 11:46:14.301188: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for memory_profile.json.gz to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.memory_profile.json.gz\n", - "Mon Sep 6 11:46:14 2021[1,4]:2021-09-06 11:46:14.310530: I tensorflow/core/profiler/rpc/client/capture_profile.cc:251] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:14 2021[1,4]:Dumped tool data for xplane.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.xplane.pb\n", - "Mon Sep 6 11:46:14 2021[1,4]:Dumped tool data for overview_page.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.overview_page.pb\n", - "Mon Sep 6 11:46:14 2021[1,4]:Dumped tool data for input_pipeline.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.input_pipeline.pb\n", - "Mon Sep 6 11:46:14 2021[1,4]:Dumped tool data for tensorflow_stats.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.tensorflow_stats.pb\n", - "Mon Sep 6 11:46:14 2021[1,4]:Dumped tool data for kernel_stats.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.kernel_stats.pb\n", - "Mon Sep 6 11:46:14 2021[1,4]:\n", - "Mon Sep 6 11:46:14 2021[1,3]:2021-09-06 11:46:14.330532: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:14 2021[1,3]:\n", - "Mon Sep 6 11:46:14 2021[1,3]:2021-09-06 11:46:14.339903: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for memory_profile.json.gz to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.memory_profile.json.gz\n", - "Mon Sep 6 11:46:14 2021[1,3]:2021-09-06 11:46:14.347180: I tensorflow/core/profiler/rpc/client/capture_profile.cc:251] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:14 2021[1,3]:Dumped tool data for xplane.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.xplane.pb\n", - "Mon Sep 6 11:46:14 2021[1,3]:Dumped tool data for overview_page.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.overview_page.pb\n", - "Mon Sep 6 11:46:14 2021[1,3]:Dumped tool data for input_pipeline.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.input_pipeline.pb\n", - "Mon Sep 6 11:46:14 2021[1,3]:Dumped tool data for tensorflow_stats.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.tensorflow_stats.pb\n", - "Mon Sep 6 11:46:14 2021[1,3]:Dumped tool data for kernel_stats.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.kernel_stats.pb\n", - "Mon Sep 6 11:46:14 2021[1,3]:\n", - "Mon Sep 6 11:46:14 2021[1,2]:2021-09-06 11:46:14.471034: I tensorflow/core/profiler/rpc/client/save_profile.cc:136] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:14 2021[1,2]:\n", - "Mon Sep 6 11:46:14 2021[1,2]:2021-09-06 11:46:14.477906: I tensorflow/core/profiler/rpc/client/save_profile.cc:142] Dumped gzipped tool data for memory_profile.json.gz to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.memory_profile.json.gz\n", - "Mon Sep 6 11:46:14 2021[1,2]:2021-09-06 11:46:14.491195: I tensorflow/core/profiler/rpc/client/capture_profile.cc:251] Creating directory: tf_logs/plugins/profile/2021_09_06_11_46_12\n", - "Mon Sep 6 11:46:14 2021[1,2]:Dumped tool data for xplane.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.xplane.pb\n", - "Mon Sep 6 11:46:14 2021[1,2]:Dumped tool data for overview_page.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.overview_page.pb\n", - "Mon Sep 6 11:46:14 2021[1,2]:Dumped tool data for input_pipeline.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.input_pipeline.pb\n", - "Mon Sep 6 11:46:14 2021[1,2]:Dumped tool data for tensorflow_stats.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.tensorflow_stats.pb\n", - "Mon Sep 6 11:46:14 2021[1,2]:Dumped tool data for kernel_stats.pb to tf_logs/plugins/profile/2021_09_06_11_46_12/dgx2h0194.kernel_stats.pb\n", - "Mon Sep 6 11:46:14 2021[1,2]:\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:46:16 2021[1,0]: 5/4000 [..............................]Mon Sep 6 11:46:16 2021[1,0]: - ETA: 4:44:11 - loss: 1.1971 - auc: 0.5447Mon Sep 6 11:46:21 2021[1,0]:>:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:46:21 2021[1,5]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 1.5142s vs `on_train_batch_end` time: 2.2162s). Check your callbacks.\n", - "Mon Sep 6 11:46:21 2021[1,15]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 1.4938s vs `on_train_batch_end` time: 2.2278s). Check your callbacks.\n", - "Mon Sep 6 11:46:21 2021[1,9]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 1.5230s vs `on_train_batch_end` time: 2.1673s). Check your callbacks.\n", - "Mon Sep 6 11:46:21 2021[1,12]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 1.5441s vs `on_train_batch_end` time: 2.1769s). Check your callbacks.\n", - "Mon Sep 6 11:46:21 2021[1,8]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 1.5099s vs `on_train_batch_end` time: 2.2067s). Check your callbacks.\n", - "Mon Sep 6 11:46:21 2021[1,6]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 1.4587s vs `on_train_batch_end` time: 2.2470s). Check your callbacks.\n", - "Mon Sep 6 11:46:21 2021[1,10]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 1.6162s vs `on_train_batch_end` time: 2.0815s). Check your callbacks.\n", - "Mon Sep 6 11:46:21 2021[1,4]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 1.4682s vs `on_train_batch_end` time: 2.2509s). Check your callbacks.\n", - "Mon Sep 6 11:46:21 2021[1,14]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 1.5344s vs `on_train_batch_end` time: 2.1920s). Check your callbacks.\n", - "Mon Sep 6 11:46:21 2021[1,13]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 1.5144s vs `on_train_batch_end` time: 2.2132s). Check your callbacks.\n", - "Mon Sep 6 11:46:21 2021[1,1]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 1.4933s vs `on_train_batch_end` time: 2.2237s). Check your callbacks.\n", - "Mon Sep 6 11:46:21 2021[1,3]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 1.4516s vs `on_train_batch_end` time: 2.2473s). Check your callbacks.\n", - "Mon Sep 6 11:46:21 2021[1,7]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 1.6179s vs `on_train_batch_end` time: 2.1218s). Check your callbacks.\n", - "Mon Sep 6 11:46:21 2021[1,11]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 1.5059s vs `on_train_batch_end` time: 2.2149s). Check your callbacks.\n", - "Mon Sep 6 11:46:21 2021[1,2]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 1.4030s vs `on_train_batch_end` time: 2.2793s). Check your callbacks.\n", - "Mon Sep 6 11:46:21 2021[1,0]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 1.5182s vs `on_train_batch_end` time: 2.1593s). Check your callbacks.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:46:45 2021[1,0]: 11/4000 [..............................]Mon Sep 6 11:46:45 2021[1,0]: - ETA: 5:01:18 - loss: 0.9724 - auc: 0.5378Mon Sep 6 11:46:49 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:47:41 2021[1,0]: 21/4000 [..............................]Mon Sep 6 11:47:41 2021[1,0]: - ETA: 5:37:25 - loss: 0.7117 - auc: 0.5280Mon Sep 6 11:47:48 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:48:44 2021[1,0]: 31/4000 [..............................]Mon Sep 6 11:48:44 2021[1,0]: - ETA: 6:02:07 - loss: 0.5560 - auc: 0.5265Mon Sep 6 11:48:49 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:49:44 2021[1,0]: 41/4000 [..............................]Mon Sep 6 11:49:44 2021[1,0]: - ETA: 6:10:48 - loss: 0.4611 - auc: 0.5275Mon Sep 6 11:49:51 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:50:42 2021[1,0]: 50/4000 [..............................]Mon Sep 6 11:50:42 2021[1,0]: - ETA: 6:20:05 - loss: 0.4050 - auc: 0.5309Mon Sep 6 11:50:49 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:51:41 2021[1,0]: 59/4000 [..............................]Mon Sep 6 11:51:41 2021[1,0]: - ETA: 6:26:39 - loss: 0.3650 - auc: 0.5373Mon Sep 6 11:51:48 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:52:44 2021[1,0]: 68/4000 [..............................]Mon Sep 6 11:52:44 2021[1,0]: - ETA: 6:35:30 - loss: 0.3352 - auc: 0.5462Mon Sep 6 11:52:51 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:53:40 2021[1,0]: 76/4000 [..............................]Mon Sep 6 11:53:40 2021[1,0]: - ETA: 6:41:37 - loss: 0.3143 - auc: 0.5561Mon Sep 6 11:53:47 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:54:38 2021[1,0]: 84/4000 [..............................]Mon Sep 6 11:54:38 2021[1,0]: - ETA: 6:48:12 - loss: 0.2972 - auc: 0.5662Mon Sep 6 11:54:46 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:55:44 2021[1,0]: 93/4000 [..............................]Mon Sep 6 11:55:44 2021[1,0]: - ETA: 6:53:56 - loss: 0.2815 - auc: 0.5771Mon Sep 6 11:55:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:56:37 2021[1,0]: 100/4000 [..............................]Mon Sep 6 11:56:37 2021[1,0]: - ETA: 6:58:55 - loss: 0.2710 - auc: 0.5854Mon Sep 6 11:56:45 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:57:39 2021[1,0]: 108/4000 [..............................]Mon Sep 6 11:57:39 2021[1,0]: - ETA: 7:04:18 - loss: 0.2609 - auc: 0.5939Mon Sep 6 11:57:47 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:58:42 2021[1,0]: 116/4000 [..............................]Mon Sep 6 11:58:42 2021[1,0]: - ETA: 7:09:03 - loss: 0.2521 - auc: 0.6023Mon Sep 6 11:58:49 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 11:59:37 2021[1,0]: 123/4000 [..............................]Mon Sep 6 11:59:37 2021[1,0]: - ETA: 7:13:04 - loss: 0.2452 - auc: 0.6090Mon Sep 6 11:59:45 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:00:42 2021[1,0]: 131/4000 [..............................]Mon Sep 6 12:00:42 2021[1,0]: - ETA: 7:17:40 - loss: 0.2382 - auc: 0.6162Mon Sep 6 12:00:50 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:01:39 2021[1,0]: 138/4000 [>.............................]Mon Sep 6 12:01:39 2021[1,0]: - ETA: 7:21:24 - loss: 0.2328 - auc: 0.6220Mon Sep 6 12:01:47 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:02:37 2021[1,0]: 145/4000 [>.............................]Mon Sep 6 12:02:37 2021[1,0]: - ETA: 7:25:19 - loss: 0.2279 - auc: 0.6274Mon Sep 6 12:02:46 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:03:37 2021[1,0]: 152/4000 [>.............................]Mon Sep 6 12:03:37 2021[1,0]: - ETA: 7:29:15 - loss: 0.2233 - auc: 0.6327Mon Sep 6 12:03:46 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:04:36 2021[1,0]: 159/4000 [>.............................]Mon Sep 6 12:04:36 2021[1,0]: - ETA: 7:32:18 - loss: 0.2192 - auc: 0.6377Mon Sep 6 12:04:44 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:05:43 2021[1,0]: 167/4000 [>.............................]Mon Sep 6 12:05:43 2021[1,0]: - ETA: 7:35:26 - loss: 0.2149 - auc: 0.6427Mon Sep 6 12:05:51 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:06:42 2021[1,0]: 174/4000 [>.............................]Mon Sep 6 12:06:42 2021[1,0]: - ETA: 7:38:11 - loss: 0.2115 - auc: 0.6469Mon Sep 6 12:06:51 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:07:42 2021[1,0]: 181/4000 [>.............................]Mon Sep 6 12:07:42 2021[1,0]: - ETA: 7:40:34 - loss: 0.2084 - auc: 0.6507Mon Sep 6 12:07:50 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:08:41 2021[1,0]: 188/4000 [>.............................]Mon Sep 6 12:08:41 2021[1,0]: - ETA: 7:42:41 - loss: 0.2055 - auc: 0.6545Mon Sep 6 12:08:50 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:09:40 2021[1,0]: 195/4000 [>.............................] - ETA: 7:44:22 - loss: 0.2027 - auc: 0.6581Mon Sep 6 12:09:49 2021[1,0]:Mon Sep 6 12:09:32 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:10:40 2021[1,0]: 202/4000 [>.............................]Mon Sep 6 12:10:40 2021[1,0]: - ETA: 7:46:12 - loss: 0.2002 - auc: 0.6614Mon Sep 6 12:10:48 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:11:40 2021[1,0]: 209/4000 [>.............................]Mon Sep 6 12:11:40 2021[1,0]: - ETA: 7:47:50 - loss: 0.1978 - auc: 0.6646Mon Sep 6 12:11:48 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:12:39 2021[1,0]: 216/4000 [>.............................]Mon Sep 6 12:12:39 2021[1,0]: - ETA: 7:49:20 - loss: 0.1956 - auc: 0.6675Mon Sep 6 12:12:48 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:13:40 2021[1,0]: 223/4000 [>.............................]Mon Sep 6 12:13:40 2021[1,0]: - ETA: 7:50:49 - loss: 0.1935 - auc: 0.6702Mon Sep 6 12:13:47 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:14:38 2021[1,0]: 230/4000 [>.............................]Mon Sep 6 12:14:38 2021[1,0]: - ETA: 7:51:37 - loss: 0.1916 - auc: 0.6730Mon Sep 6 12:14:47 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:15:38 2021[1,0]: 237/4000 [>.............................]Mon Sep 6 12:15:38 2021[1,0]: - ETA: 7:52:41 - loss: 0.1897 - auc: 0.6754Mon Sep 6 12:15:47 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:16:38 2021[1,0]: 244/4000 [>.............................]Mon Sep 6 12:16:38 2021[1,0]: - ETA: 7:53:46 - loss: 0.1880 - auc: 0.6779Mon Sep 6 12:16:47 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:17:38 2021[1,0]: 251/4000 [>.............................]Mon Sep 6 12:17:38 2021[1,0]: - ETA: 7:54:34 - loss: 0.1864 - auc: 0.6802Mon Sep 6 12:17:47 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:18:37 2021[1,0]: 258/4000 [>.............................]Mon Sep 6 12:18:37 2021[1,0]: - ETA: 7:55:03 - loss: 0.1848 - auc: 0.6824Mon Sep 6 12:18:46 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:19:37 2021[1,0]: 265/4000 [>.............................]Mon Sep 6 12:19:37 2021[1,0]: - ETA: 7:55:43 - loss: 0.1832 - auc: 0.6846Mon Sep 6 12:19:45 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - " 272/4000 [=>............................]Mon Sep 6 12:20:37 2021[1,0]: - ETA: 7:56:22 - loss: 0.1818 - auc: 0.6867Mon Sep 6 12:20:46 2021[1,0]:Mon Sep 6 12:20:20 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - " 279/4000 [=>............................]Mon Sep 6 12:21:38 2021[1,0]: - ETA: 7:57:08 - loss: 0.1804 - auc: 0.6887Mon Sep 6 12:21:47 2021[1,0]:Mon Sep 6 12:21:29 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:22:38 2021[1,0]: 286/4000 [=>............................]Mon Sep 6 12:22:38 2021[1,0]: - ETA: 7:57:26 - loss: 0.1791 - auc: 0.6906Mon Sep 6 12:22:46 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:23:38 2021[1,0]: 293/4000 [=>............................]Mon Sep 6 12:23:38 2021[1,0]: - ETA: 7:57:59 - loss: 0.1779 - auc: 0.6924Mon Sep 6 12:23:47 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:24:40 2021[1,0]: 300/4000 [=>............................]Mon Sep 6 12:24:40 2021[1,0]: - ETA: 7:58:35 - loss: 0.1767 - auc: 0.6941Mon Sep 6 12:24:48 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:25:40 2021[1,0]: 307/4000 [=>............................]Mon Sep 6 12:25:40 2021[1,0]: - ETA: 7:58:54 - loss: 0.1756 - auc: 0.6957Mon Sep 6 12:25:49 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:26:40 2021[1,0]: 314/4000 [=>............................]Mon Sep 6 12:26:40 2021[1,0]: - ETA: 7:59:03 - loss: 0.1745 - auc: 0.6973Mon Sep 6 12:26:48 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:27:40 2021[1,0]: 321/4000 [=>............................]Mon Sep 6 12:27:40 2021[1,0]: - ETA: 7:59:04 - loss: 0.1735 - auc: 0.6988Mon Sep 6 12:27:48 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:28:40 2021[1,0]: 328/4000 [=>............................]Mon Sep 6 12:28:40 2021[1,0]: - ETA: 7:59:18 - loss: 0.1726 - auc: 0.7003Mon Sep 6 12:28:49 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:29:42 2021[1,0]: 335/4000 [=>............................]Mon Sep 6 12:29:42 2021[1,0]: - ETA: 7:59:35 - loss: 0.1716 - auc: 0.7017Mon Sep 6 12:29:50 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:30:42 2021[1,0]: 342/4000 [=>............................]Mon Sep 6 12:30:42 2021[1,0]: - ETA: 7:59:39 - loss: 0.1707 - auc: 0.7031Mon Sep 6 12:30:50 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:31:42 2021[1,0]: 349/4000 [=>............................]Mon Sep 6 12:31:42 2021[1,0]: - ETA: 7:59:34 - loss: 0.1698 - auc: 0.7044Mon Sep 6 12:31:51 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:32:43 2021[1,0]: 356/4000 [=>............................]Mon Sep 6 12:32:43 2021[1,0]: - ETA: 7:59:36 - loss: 0.1689 - auc: 0.7058Mon Sep 6 12:32:51 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:33:36 2021[1,0]: 362/4000 [=>............................]Mon Sep 6 12:33:36 2021[1,0]: - ETA: 7:59:43 - loss: 0.1683 - auc: 0.7069Mon Sep 6 12:33:44 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:34:37 2021[1,0]: 369/4000 [=>............................]Mon Sep 6 12:34:37 2021[1,0]: - ETA: 7:59:48 - loss: 0.1674 - auc: 0.7082Mon Sep 6 12:34:46 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:35:38 2021[1,0]: 376/4000 [=>............................]Mon Sep 6 12:35:38 2021[1,0]: - ETA: 7:59:40 - loss: 0.1667 - auc: 0.7093Mon Sep 6 12:35:46 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:36:37 2021[1,0]: 383/4000 [=>............................]Mon Sep 6 12:36:37 2021[1,0]: - ETA: 7:59:26 - loss: 0.1660 - auc: 0.7106Mon Sep 6 12:36:46 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:37:39 2021[1,0]: 390/4000 [=>............................]Mon Sep 6 12:37:39 2021[1,0]: - ETA: 7:59:27 - loss: 0.1653 - auc: 0.7117Mon Sep 6 12:37:48 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:38:40 2021[1,0]: 397/4000 [=>............................]Mon Sep 6 12:38:40 2021[1,0]: - ETA: 7:59:20 - loss: 0.1646 - auc: 0.7127Mon Sep 6 12:38:49 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:39:42 2021[1,0]: 404/4000 [==>...........................]Mon Sep 6 12:39:42 2021[1,0]: - ETA: 7:59:13 - loss: 0.1640 - auc: 0.7137Mon Sep 6 12:39:50 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:40:42 2021[1,0]: 411/4000 [==>...........................]Mon Sep 6 12:40:42 2021[1,0]: - ETA: 7:58:54 - loss: 0.1633 - auc: 0.7148Mon Sep 6 12:40:50 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:41:44 2021[1,0]: 418/4000 [==>...........................]Mon Sep 6 12:41:44 2021[1,0]: - ETA: 7:58:49 - loss: 0.1627 - auc: 0.7157Mon Sep 6 12:41:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:42:36 2021[1,0]: 424/4000 [==>...........................]Mon Sep 6 12:42:36 2021[1,0]: - ETA: 7:58:35 - loss: 0.1622 - auc: 0.7165Mon Sep 6 12:42:45 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:43:36 2021[1,0]: 431/4000 [==>...........................]Mon Sep 6 12:43:36 2021[1,0]: - ETA: 7:58:09 - loss: 0.1617 - auc: 0.7174Mon Sep 6 12:43:45 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:44:39 2021[1,0]: 438/4000 [==>...........................]Mon Sep 6 12:44:39 2021[1,0]: - ETA: 7:58:06 - loss: 0.1611 - auc: 0.7183Mon Sep 6 12:44:47 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:45:40 2021[1,0]: 445/4000 [==>...........................]Mon Sep 6 12:45:40 2021[1,0]: - ETA: 7:57:50 - loss: 0.1606 - auc: 0.7192Mon Sep 6 12:45:49 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:46:42 2021[1,0]: 452/4000 [==>...........................]Mon Sep 6 12:46:42 2021[1,0]: - ETA: 7:57:34 - loss: 0.1601 - auc: 0.7201Mon Sep 6 12:46:50 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:47:44 2021[1,0]: 459/4000 [==>...........................]Mon Sep 6 12:47:44 2021[1,0]: - ETA: 7:57:19 - loss: 0.1596 - auc: 0.7209Mon Sep 6 12:47:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:48:36 2021[1,0]: 465/4000 [==>...........................]Mon Sep 6 12:48:36 2021[1,0]: - ETA: 7:56:58 - loss: 0.1591 - auc: 0.7215Mon Sep 6 12:48:44 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:49:37 2021[1,0]: 472/4000 [==>...........................]Mon Sep 6 12:49:37 2021[1,0]: - ETA: 7:56:33 - loss: 0.1587 - auc: 0.7223Mon Sep 6 12:49:45 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:50:36 2021[1,0]: 479/4000 [==>...........................]Mon Sep 6 12:50:36 2021[1,0]: - ETA: 7:55:58 - loss: 0.1582 - auc: 0.7231Mon Sep 6 12:50:45 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:51:37 2021[1,0]: 486/4000 [==>...........................]Mon Sep 6 12:51:37 2021[1,0]: - ETA: 7:55:26 - loss: 0.1577 - auc: 0.7239Mon Sep 6 12:51:45 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:52:38 2021[1,0]: 493/4000 [==>...........................]Mon Sep 6 12:52:38 2021[1,0]: - ETA: 7:55:02 - loss: 0.1573 - auc: 0.7246Mon Sep 6 12:52:47 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:53:40 2021[1,0]: 500/4000 [==>...........................]Mon Sep 6 12:53:40 2021[1,0]: - ETA: 7:54:37 - loss: 0.1568 - auc: 0.7253Mon Sep 6 12:53:48 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:54:40 2021[1,0]: 507/4000 [==>...........................]Mon Sep 6 12:54:40 2021[1,0]: - ETA: 7:54:04 - loss: 0.1564 - auc: 0.7261Mon Sep 6 12:54:48 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:55:40 2021[1,0]: 514/4000 [==>...........................]Mon Sep 6 12:55:40 2021[1,0]: - ETA: 7:53:27 - loss: 0.1560 - auc: 0.7268Mon Sep 6 12:55:49 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:56:40 2021[1,0]: 521/4000 [==>...........................]Mon Sep 6 12:56:40 2021[1,0]: - ETA: 7:52:50 - loss: 0.1556 - auc: 0.7274Mon Sep 6 12:56:48 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:57:39 2021[1,0]: 528/4000 [==>...........................]Mon Sep 6 12:57:39 2021[1,0]: - ETA: 7:52:08 - loss: 0.1553 - auc: 0.7281Mon Sep 6 12:57:48 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:58:40 2021[1,0]: 535/4000 [===>..........................]Mon Sep 6 12:58:40 2021[1,0]: - ETA: 7:51:35 - loss: 0.1549 - auc: 0.7287Mon Sep 6 12:58:49 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 12:59:41 2021[1,0]: 542/4000 [===>..........................]Mon Sep 6 12:59:41 2021[1,0]: - ETA: 7:51:05 - loss: 0.1545 - auc: 0.7293Mon Sep 6 12:59:50 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:00:42 2021[1,0]: 549/4000 [===>..........................]Mon Sep 6 13:00:42 2021[1,0]: - ETA: 7:50:26 - loss: 0.1542 - auc: 0.7299Mon Sep 6 13:00:50 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - " 556/4000 [===>..........................]Mon Sep 6 13:01:41 2021[1,0]: - ETA: 7:49:40 - loss: 0.1538 - auc: 0.7305Mon Sep 6 13:01:49 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:02:41 2021[1,0]: 563/4000 [===>..........................]Mon Sep 6 13:02:41 2021[1,0]: - ETA: 7:49:01 - loss: 0.1535 - auc: 0.7311Mon Sep 6 13:02:50 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:03:40 2021[1,0]: 570/4000 [===>..........................]Mon Sep 6 13:03:40 2021[1,0]: - ETA: 7:48:17 - loss: 0.1531 - auc: 0.7316Mon Sep 6 13:03:49 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:04:38 2021[1,0]: 577/4000 [===>..........................]Mon Sep 6 13:04:38 2021[1,0]: - ETA: 7:47:19 - loss: 0.1528 - auc: 0.7322Mon Sep 6 13:04:46 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:05:42 2021[1,0]: 585/4000 [===>..........................]Mon Sep 6 13:05:42 2021[1,0]: - ETA: 7:46:07 - loss: 0.1524 - auc: 0.7328Mon Sep 6 13:05:51 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:06:43 2021[1,0]: 592/4000 [===>..........................]Mon Sep 6 13:06:43 2021[1,0]: - ETA: 7:45:28 - loss: 0.1521 - auc: 0.7333Mon Sep 6 13:06:51 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:07:43 2021[1,0]: 599/4000 [===>..........................]Mon Sep 6 13:07:43 2021[1,0]: - ETA: 7:44:49 - loss: 0.1518 - auc: 0.7338Mon Sep 6 13:07:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:08:44 2021[1,0]: 606/4000 [===>..........................]Mon Sep 6 13:08:44 2021[1,0]: - ETA: 7:44:09 - loss: 0.1515 - auc: 0.7343Mon Sep 6 13:08:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:09:43 2021[1,0]: 613/4000 [===>..........................]Mon Sep 6 13:09:43 2021[1,0]: - ETA: 7:43:22 - loss: 0.1513 - auc: 0.7348Mon Sep 6 13:09:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:10:41 2021[1,0]: 620/4000 [===>..........................]Mon Sep 6 13:10:41 2021[1,0]: - ETA: 7:42:25 - loss: 0.1510 - auc: 0.7353Mon Sep 6 13:10:49 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:11:38 2021[1,0]: 627/4000 [===>..........................]Mon Sep 6 13:11:38 2021[1,0]: - ETA: 7:41:25 - loss: 0.1507 - auc: 0.7358Mon Sep 6 13:11:46 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:12:44 2021[1,0]: 635/4000 [===>..........................]Mon Sep 6 13:12:44 2021[1,0]: - ETA: 7:40:24 - loss: 0.1504 - auc: 0.7363Mon Sep 6 13:12:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:13:44 2021[1,0]: 642/4000 [===>..........................]Mon Sep 6 13:13:44 2021[1,0]: - ETA: 7:39:39 - loss: 0.1502 - auc: 0.7367Mon Sep 6 13:13:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:14:45 2021[1,0]: 649/4000 [===>..........................]Mon Sep 6 13:14:45 2021[1,0]: - ETA: 7:38:58 - loss: 0.1499 - auc: 0.7372Mon Sep 6 13:14:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - " 656/4000 [===>..........................]Mon Sep 6 13:15:45 2021[1,0]: - ETA: 7:38:14 - loss: 0.1497 - auc: 0.7377Mon Sep 6 13:15:54 2021[1,0]:Mon Sep 6 13:15:28 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:16:44 2021[1,0]: 663/4000 [===>..........................]Mon Sep 6 13:16:44 2021[1,0]: - ETA: 7:37:26 - loss: 0.1494 - auc: 0.7381Mon Sep 6 13:16:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:17:36 2021[1,0]: 669/4000 [====>.........................]Mon Sep 6 13:17:36 2021[1,0]: - ETA: 7:36:48 - loss: 0.1492 - auc: 0.7385Mon Sep 6 13:17:45 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - " 676/4000 [====>.........................]Mon Sep 6 13:18:38 2021[1,0]: - ETA: 7:36:12 - loss: 0.1490 - auc: 0.7389Mon Sep 6 13:18:47 2021[1,0]:Mon Sep 6 13:18:38 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - " 684/4000 [====>.........................]Mon Sep 6 13:19:45 2021[1,0]: - ETA: 7:35:13 - loss: 0.1487 - auc: 0.7394Mon Sep 6 13:19:54 2021[1,0]:Mon Sep 6 13:19:37 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:20:45 2021[1,0]: 691/4000 [====>.........................]Mon Sep 6 13:20:45 2021[1,0]: - ETA: 7:34:27 - loss: 0.1485 - auc: 0.7398Mon Sep 6 13:20:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:21:43 2021[1,0]: 698/4000 [====>.........................]Mon Sep 6 13:21:43 2021[1,0]: - ETA: 7:33:32 - loss: 0.1483 - auc: 0.7402Mon Sep 6 13:21:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:22:41 2021[1,0]: 705/4000 [====>.........................]Mon Sep 6 13:22:41 2021[1,0]: - ETA: 7:32:35 - loss: 0.1481 - auc: 0.7406Mon Sep 6 13:22:49 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:23:46 2021[1,0]: 713/4000 [====>.........................]Mon Sep 6 13:23:46 2021[1,0]: - ETA: 7:31:21 - loss: 0.1478 - auc: 0.7410Mon Sep 6 13:23:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:24:43 2021[1,0]: 720/4000 [====>.........................]Mon Sep 6 13:24:43 2021[1,0]: - ETA: 7:30:24 - loss: 0.1476 - auc: 0.7414Mon Sep 6 13:24:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:25:43 2021[1,0]: 727/4000 [====>.........................]Mon Sep 6 13:25:43 2021[1,0]: - ETA: 7:29:35 - loss: 0.1474 - auc: 0.7417Mon Sep 6 13:25:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:26:44 2021[1,0]: 734/4000 [====>.........................]Mon Sep 6 13:26:44 2021[1,0]: - ETA: 7:28:51 - loss: 0.1472 - auc: 0.7421Mon Sep 6 13:26:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:27:37 2021[1,0]: 740/4000 [====>.........................]Mon Sep 6 13:27:37 2021[1,0]: - ETA: 7:28:19 - loss: 0.1470 - auc: 0.7424Mon Sep 6 13:27:45 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:28:38 2021[1,0]: 747/4000 [====>.........................]Mon Sep 6 13:28:38 2021[1,0]: - ETA: 7:27:33 - loss: 0.1469 - auc: 0.7427Mon Sep 6 13:28:47 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:29:39 2021[1,0]: 754/4000 [====>.........................]Mon Sep 6 13:29:39 2021[1,0]: - ETA: 7:26:52 - loss: 0.1467 - auc: 0.7431Mon Sep 6 13:29:48 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:30:38 2021[1,0]: 761/4000 [====>.........................]Mon Sep 6 13:30:38 2021[1,0]: - ETA: 7:25:59 - loss: 0.1465 - auc: 0.7434Mon Sep 6 13:30:47 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - " 768/4000 [====>.........................]Mon Sep 6 13:31:38 2021[1,0]: - ETA: 7:25:08 - loss: 0.1463 - auc: 0.7437Mon Sep 6 13:31:46 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - " 775/4000 [====>.........................]Mon Sep 6 13:32:37 2021[1,0]: - ETA: 7:24:18 - loss: 0.1461 - auc: 0.7441Mon Sep 6 13:32:46 2021[1,0]:Mon Sep 6 13:32:29 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:33:37 2021[1,0]: 782/4000 [====>.........................]Mon Sep 6 13:33:37 2021[1,0]: - ETA: 7:23:28 - loss: 0.1459 - auc: 0.7444Mon Sep 6 13:33:46 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:34:37 2021[1,0]: 789/4000 [====>.........................]Mon Sep 6 13:34:37 2021[1,0]: - ETA: 7:22:40 - loss: 0.1458 - auc: 0.7447Mon Sep 6 13:34:46 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:35:45 2021[1,0]: 797/4000 [====>.........................]Mon Sep 6 13:35:45 2021[1,0]: - ETA: 7:21:40 - loss: 0.1456 - auc: 0.7451Mon Sep 6 13:35:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - " 803/4000 [=====>........................]Mon Sep 6 13:36:38 2021[1,0]: - ETA: 7:21:02 - loss: 0.1454 - auc: 0.7454Mon Sep 6 13:36:46 2021[1,0]:Mon Sep 6 13:36:20 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:37:46 2021[1,0]: 811/4000 [=====>........................]Mon Sep 6 13:37:46 2021[1,0]: - ETA: 7:20:03 - loss: 0.1452 - auc: 0.7457Mon Sep 6 13:37:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:38:45 2021[1,0]: 818/4000 [=====>........................]Mon Sep 6 13:38:45 2021[1,0]: - ETA: 7:19:11 - loss: 0.1451 - auc: 0.7460Mon Sep 6 13:38:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:39:43 2021[1,0]: 825/4000 [=====>........................]Mon Sep 6 13:39:43 2021[1,0]: - ETA: 7:18:13 - loss: 0.1449 - auc: 0.7463Mon Sep 6 13:39:51 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:40:40 2021[1,0]: 832/4000 [=====>........................]Mon Sep 6 13:40:40 2021[1,0]: - ETA: 7:17:10 - loss: 0.1448 - auc: 0.7466Mon Sep 6 13:40:48 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - " 839/4000 [=====>........................]Mon Sep 6 13:41:39 2021[1,0]: - ETA: 7:16:15 - loss: 0.1446 - auc: 0.7469Mon Sep 6 13:41:47 2021[1,0]:Mon Sep 6 13:41:21 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:42:39 2021[1,0]: 846/4000 [=====>........................]Mon Sep 6 13:42:39 2021[1,0]: - ETA: 7:15:25 - loss: 0.1444 - auc: 0.7472Mon Sep 6 13:42:47 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:43:44 2021[1,0]: 854/4000 [=====>........................]Mon Sep 6 13:43:44 2021[1,0]: - ETA: 7:14:13 - loss: 0.1443 - auc: 0.7475Mon Sep 6 13:43:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:44:42 2021[1,0]: 861/4000 [=====>........................]Mon Sep 6 13:44:42 2021[1,0]: - ETA: 7:13:19 - loss: 0.1441 - auc: 0.7478Mon Sep 6 13:44:51 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:45:43 2021[1,0]: 868/4000 [=====>........................]Mon Sep 6 13:45:43 2021[1,0]: - ETA: 7:12:32 - loss: 0.1440 - auc: 0.7481Mon Sep 6 13:45:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:46:43 2021[1,0]: 875/4000 [=====>........................]Mon Sep 6 13:46:43 2021[1,0]: - ETA: 7:11:40 - loss: 0.1438 - auc: 0.7483Mon Sep 6 13:46:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:47:43 2021[1,0]: 882/4000 [=====>........................]Mon Sep 6 13:47:43 2021[1,0]: - ETA: 7:10:49 - loss: 0.1437 - auc: 0.7486Mon Sep 6 13:47:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:48:44 2021[1,0]: 889/4000 [=====>........................]Mon Sep 6 13:48:44 2021[1,0]: - ETA: 7:10:00 - loss: 0.1435 - auc: 0.7489Mon Sep 6 13:48:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:49:43 2021[1,0]: 896/4000 [=====>........................]Mon Sep 6 13:49:43 2021[1,0]: - ETA: 7:09:06 - loss: 0.1434 - auc: 0.7491Mon Sep 6 13:49:51 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:50:41 2021[1,0]: 903/4000 [=====>........................]Mon Sep 6 13:50:41 2021[1,0]: - ETA: 7:08:08 - loss: 0.1433 - auc: 0.7494Mon Sep 6 13:50:50 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:51:39 2021[1,0]: 910/4000 [=====>........................]Mon Sep 6 13:51:39 2021[1,0]: - ETA: 7:07:11 - loss: 0.1431 - auc: 0.7496Mon Sep 6 13:51:48 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:52:41 2021[1,0]: 917/4000 [=====>........................]Mon Sep 6 13:52:41 2021[1,0]: - ETA: 7:06:25 - loss: 0.1430 - auc: 0.7498Mon Sep 6 13:52:50 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:53:42 2021[1,0]: 924/4000 [=====>........................]Mon Sep 6 13:53:42 2021[1,0]: - ETA: 7:05:35 - loss: 0.1429 - auc: 0.7501Mon Sep 6 13:53:50 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - " 931/4000 [=====>........................]Mon Sep 6 13:54:43 2021[1,0]: - ETA: 7:04:47 - loss: 0.1427 - auc: 0.7503Mon Sep 6 13:54:51 2021[1,0]:Mon Sep 6 13:54:34 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:55:43 2021[1,0]: 938/4000 [======>.......................]Mon Sep 6 13:55:43 2021[1,0]: - ETA: 7:03:56 - loss: 0.1426 - auc: 0.7505Mon Sep 6 13:55:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:56:43 2021[1,0]: 945/4000 [======>.......................]Mon Sep 6 13:56:43 2021[1,0]: - ETA: 7:03:04 - loss: 0.1425 - auc: 0.7508Mon Sep 6 13:56:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - " 952/4000 [======>.......................]Mon Sep 6 13:57:43 2021[1,0]: - ETA: 7:02:11 - loss: 0.1424 - auc: 0.7510Mon Sep 6 13:57:51 2021[1,0]:Mon Sep 6 13:57:43 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:58:42 2021[1,0]: 959/4000 [======>.......................]Mon Sep 6 13:58:42 2021[1,0]: - ETA: 7:01:16 - loss: 0.1422 - auc: 0.7512Mon Sep 6 13:58:51 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 13:59:42 2021[1,0]: 966/4000 [======>.......................]Mon Sep 6 13:59:42 2021[1,0]: - ETA: 7:00:24 - loss: 0.1421 - auc: 0.7515Mon Sep 6 13:59:51 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - " 973/4000 [======>.......................]Mon Sep 6 14:00:40 2021[1,0]: - ETA: 6:59:25 - loss: 0.1420 - auc: 0.7517Mon Sep 6 14:00:48 2021[1,0]:Mon Sep 6 14:00:40 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:01:40 2021[1,0]: 980/4000 [======>.......................]Mon Sep 6 14:01:40 2021[1,0]: - ETA: 6:58:30 - loss: 0.1419 - auc: 0.7519Mon Sep 6 14:01:48 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:02:39 2021[1,0]: 987/4000 [======>.......................]Mon Sep 6 14:02:39 2021[1,0]: - ETA: 6:57:34 - loss: 0.1418 - auc: 0.7521Mon Sep 6 14:02:47 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:03:46 2021[1,0]: 995/4000 [======>.......................]Mon Sep 6 14:03:46 2021[1,0]: - ETA: 6:56:32 - loss: 0.1416 - auc: 0.7524Mon Sep 6 14:03:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:04:45 2021[1,0]:1002/4000 [======>.......................]Mon Sep 6 14:04:45 2021[1,0]: - ETA: 6:55:34 - loss: 0.1415 - auc: 0.7526Mon Sep 6 14:04:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:05:45 2021[1,0]:1009/4000 [======>.......................]Mon Sep 6 14:05:45 2021[1,0]: - ETA: 6:54:42 - loss: 0.1414 - auc: 0.7528Mon Sep 6 14:05:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:06:45 2021[1,0]:1016/4000 [======>.......................]Mon Sep 6 14:06:45 2021[1,0]: - ETA: 6:53:49 - loss: 0.1413 - auc: 0.7530Mon Sep 6 14:06:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:07:45 2021[1,0]:1023/4000 [======>.......................]Mon Sep 6 14:07:45 2021[1,0]: - ETA: 6:52:55 - loss: 0.1412 - auc: 0.7532Mon Sep 6 14:07:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1030/4000 [======>.......................]Mon Sep 6 14:08:45 2021[1,0]: - ETA: 6:52:02 - loss: 0.1411 - auc: 0.7534Mon Sep 6 14:08:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:09:44 2021[1,0]:1037/4000 [======>.......................]Mon Sep 6 14:09:44 2021[1,0]: - ETA: 6:51:06 - loss: 0.1410 - auc: 0.7536Mon Sep 6 14:09:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1044/4000 [======>.......................]Mon Sep 6 14:10:42 2021[1,0]: - ETA: 6:50:08 - loss: 0.1409 - auc: 0.7538Mon Sep 6 14:10:51 2021[1,0]:Mon Sep 6 14:10:25 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:11:41 2021[1,0]:1051/4000 [======>.......................]Mon Sep 6 14:11:41 2021[1,0]: - ETA: 6:49:12 - loss: 0.1408 - auc: 0.7540Mon Sep 6 14:11:50 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:12:41 2021[1,0]:1058/4000 [======>.......................]Mon Sep 6 14:12:41 2021[1,0]: - ETA: 6:48:18 - loss: 0.1407 - auc: 0.7542Mon Sep 6 14:12:50 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:13:42 2021[1,0]:1065/4000 [======>.......................]Mon Sep 6 14:13:42 2021[1,0]: - ETA: 6:47:28 - loss: 0.1406 - auc: 0.7544Mon Sep 6 14:13:51 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1072/4000 [=======>......................]Mon Sep 6 14:14:43 2021[1,0]: - ETA: 6:46:35 - loss: 0.1405 - auc: 0.7546Mon Sep 6 14:14:51 2021[1,0]:Mon Sep 6 14:14:43 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:15:42 2021[1,0]:1079/4000 [=======>......................]Mon Sep 6 14:15:42 2021[1,0]: - ETA: 6:45:40 - loss: 0.1404 - auc: 0.7548Mon Sep 6 14:15:51 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1086/4000 [=======>......................]Mon Sep 6 14:16:42 2021[1,0]: - ETA: 6:44:46 - loss: 0.1403 - auc: 0.7550Mon Sep 6 14:16:51 2021[1,0]:Mon Sep 6 14:16:42 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:17:42 2021[1,0]:1093/4000 [=======>......................]Mon Sep 6 14:17:42 2021[1,0]: - ETA: 6:43:52 - loss: 0.1402 - auc: 0.7551Mon Sep 6 14:17:51 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:18:43 2021[1,0]:1100/4000 [=======>......................]Mon Sep 6 14:18:43 2021[1,0]: - ETA: 6:42:59 - loss: 0.1402 - auc: 0.7553Mon Sep 6 14:18:51 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:19:43 2021[1,0]:1107/4000 [=======>......................]Mon Sep 6 14:19:43 2021[1,0]: - ETA: 6:42:06 - loss: 0.1401 - auc: 0.7555Mon Sep 6 14:19:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:20:44 2021[1,0]:1114/4000 [=======>......................]Mon Sep 6 14:20:44 2021[1,0]: - ETA: 6:41:13 - loss: 0.1400 - auc: 0.7556Mon Sep 6 14:20:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:21:45 2021[1,0]:1121/4000 [=======>......................]Mon Sep 6 14:21:45 2021[1,0]: - ETA: 6:40:23 - loss: 0.1399 - auc: 0.7558Mon Sep 6 14:21:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:22:45 2021[1,0]:1128/4000 [=======>......................]Mon Sep 6 14:22:45 2021[1,0]: - ETA: 6:39:29 - loss: 0.1398 - auc: 0.7560Mon Sep 6 14:22:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:23:46 2021[1,0]:1135/4000 [=======>......................]Mon Sep 6 14:23:46 2021[1,0]: - ETA: 6:38:37 - loss: 0.1397 - auc: 0.7562Mon Sep 6 14:23:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:24:39 2021[1,0]:1141/4000 [=======>......................]Mon Sep 6 14:24:39 2021[1,0]: - ETA: 6:37:54 - loss: 0.1396 - auc: 0.7563Mon Sep 6 14:24:48 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:25:40 2021[1,0]:1148/4000 [=======>......................]Mon Sep 6 14:25:40 2021[1,0]: - ETA: 6:37:01 - loss: 0.1395 - auc: 0.7565Mon Sep 6 14:25:48 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1156/4000 [=======>......................]Mon Sep 6 14:26:46 2021[1,0]: - ETA: 6:35:52 - loss: 0.1394 - auc: 0.7567Mon Sep 6 14:26:54 2021[1,0]:Mon Sep 6 14:26:46 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:27:46 2021[1,0]:1163/4000 [=======>......................]Mon Sep 6 14:27:46 2021[1,0]: - ETA: 6:34:58 - loss: 0.1393 - auc: 0.7568Mon Sep 6 14:27:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1170/4000 [=======>......................]Mon Sep 6 14:28:46 2021[1,0]: - ETA: 6:34:04 - loss: 0.1393 - auc: 0.7570Mon Sep 6 14:28:55 2021[1,0]:Mon Sep 6 14:28:38 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:29:47 2021[1,0]:1177/4000 [=======>......................]Mon Sep 6 14:29:47 2021[1,0]: - ETA: 6:33:10 - loss: 0.1392 - auc: 0.7571Mon Sep 6 14:29:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:30:47 2021[1,0]:1184/4000 [=======>......................]Mon Sep 6 14:30:47 2021[1,0]: - ETA: 6:32:16 - loss: 0.1391 - auc: 0.7573Mon Sep 6 14:30:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1190/4000 [=======>......................]Mon Sep 6 14:31:39 2021[1,0]: - ETA: 6:31:30 - loss: 0.1390 - auc: 0.7575Mon Sep 6 14:31:47 2021[1,0]:Mon Sep 6 14:31:30 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:32:39 2021[1,0]:1197/4000 [=======>......................]Mon Sep 6 14:32:39 2021[1,0]: - ETA: 6:30:35 - loss: 0.1389 - auc: 0.7576Mon Sep 6 14:32:48 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:33:38 2021[1,0]:1204/4000 [========>.....................]Mon Sep 6 14:33:38 2021[1,0]: - ETA: 6:29:38 - loss: 0.1389 - auc: 0.7578Mon Sep 6 14:33:47 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:34:39 2021[1,0]:1211/4000 [========>.....................]Mon Sep 6 14:34:39 2021[1,0]: - ETA: 6:28:45 - loss: 0.1388 - auc: 0.7579Mon Sep 6 14:34:48 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:35:45 2021[1,0]:1219/4000 [========>.....................] - ETA: 6:27:36 - loss: 0.1387 - auc: 0.7581Mon Sep 6 14:35:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:36:44 2021[1,0]:1226/4000 [========>.....................]Mon Sep 6 14:36:44 2021[1,0]: - ETA: 6:26:39 - loss: 0.1386 - auc: 0.7582Mon Sep 6 14:36:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1233/4000 [========>.....................]Mon Sep 6 14:37:45 2021[1,0]: - ETA: 6:25:45 - loss: 0.1386 - auc: 0.7583Mon Sep 6 14:37:53 2021[1,0]:Mon Sep 6 14:37:19 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:38:44 2021[1,0]:1240/4000 [========>.....................]Mon Sep 6 14:38:44 2021[1,0]: - ETA: 6:24:47 - loss: 0.1385 - auc: 0.7585Mon Sep 6 14:38:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1247/4000 [========>.....................]Mon Sep 6 14:39:40 2021[1,0]: - ETA: 6:23:43 - loss: 0.1384 - auc: 0.7586Mon Sep 6 14:39:48 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:40:45 2021[1,0]:1255/4000 [========>.....................]Mon Sep 6 14:40:45 2021[1,0]: - ETA: 6:22:32 - loss: 0.1383 - auc: 0.7588Mon Sep 6 14:40:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:41:45 2021[1,0]:1262/4000 [========>.....................]Mon Sep 6 14:41:45 2021[1,0]: - ETA: 6:21:38 - loss: 0.1383 - auc: 0.7589Mon Sep 6 14:41:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:42:43 2021[1,0]:1269/4000 [========>.....................]Mon Sep 6 14:42:43 2021[1,0]: - ETA: 6:20:38 - loss: 0.1382 - auc: 0.7591Mon Sep 6 14:42:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1276/4000 [========>.....................]Mon Sep 6 14:43:43 2021[1,0]: - ETA: 6:19:41 - loss: 0.1381 - auc: 0.7592Mon Sep 6 14:43:51 2021[1,0]:Mon Sep 6 14:43:43 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:44:43 2021[1,0]:1283/4000 [========>.....................]Mon Sep 6 14:44:43 2021[1,0]: - ETA: 6:18:47 - loss: 0.1381 - auc: 0.7594Mon Sep 6 14:44:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:45:43 2021[1,0]:1290/4000 [========>.....................]Mon Sep 6 14:45:43 2021[1,0]: - ETA: 6:17:50 - loss: 0.1380 - auc: 0.7595Mon Sep 6 14:45:51 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:46:43 2021[1,0]:1297/4000 [========>.....................]Mon Sep 6 14:46:43 2021[1,0]: - ETA: 6:16:55 - loss: 0.1379 - auc: 0.7596Mon Sep 6 14:46:51 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1304/4000 [========>.....................]Mon Sep 6 14:47:43 2021[1,0]: - ETA: 6:16:01 - loss: 0.1379 - auc: 0.7598Mon Sep 6 14:47:52 2021[1,0]:Mon Sep 6 14:47:43 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:48:43 2021[1,0]:1311/4000 [========>.....................]Mon Sep 6 14:48:43 2021[1,0]: - ETA: 6:15:04 - loss: 0.1378 - auc: 0.7599Mon Sep 6 14:48:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1318/4000 [========>.....................]Mon Sep 6 14:49:43 2021[1,0]: - ETA: 6:14:09 - loss: 0.1377 - auc: 0.7601Mon Sep 6 14:49:52 2021[1,0]:Mon Sep 6 14:49:26 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:50:42 2021[1,0]:1325/4000 [========>.....................]Mon Sep 6 14:50:42 2021[1,0]: - ETA: 6:13:10 - loss: 0.1377 - auc: 0.7602Mon Sep 6 14:50:50 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:51:41 2021[1,0]:1332/4000 [========>.....................]Mon Sep 6 14:51:41 2021[1,0]: - ETA: 6:12:13 - loss: 0.1376 - auc: 0.7603Mon Sep 6 14:51:50 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:52:40 2021[1,0]:1339/4000 [=========>....................]Mon Sep 6 14:52:40 2021[1,0]: - ETA: 6:11:14 - loss: 0.1375 - auc: 0.7605Mon Sep 6 14:52:48 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:53:46 2021[1,0]:1347/4000 [=========>....................]Mon Sep 6 14:53:46 2021[1,0]: - ETA: 6:10:06 - loss: 0.1375 - auc: 0.7606Mon Sep 6 14:53:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:54:46 2021[1,0]:1354/4000 [=========>....................]Mon Sep 6 14:54:46 2021[1,0]: - ETA: 6:09:10 - loss: 0.1374 - auc: 0.7608Mon Sep 6 14:54:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:55:44 2021[1,0]:1361/4000 [=========>....................]Mon Sep 6 14:55:44 2021[1,0]: - ETA: 6:08:11 - loss: 0.1373 - auc: 0.7609Mon Sep 6 14:55:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:56:43 2021[1,0]:1368/4000 [=========>....................]Mon Sep 6 14:56:43 2021[1,0]: - ETA: 6:07:12 - loss: 0.1373 - auc: 0.7610Mon Sep 6 14:56:51 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1375/4000 [=========>....................]Mon Sep 6 14:57:41 2021[1,0]: - ETA: 6:06:13 - loss: 0.1372 - auc: 0.7611Mon Sep 6 14:57:50 2021[1,0]:Mon Sep 6 14:57:33 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1382/4000 [=========>....................]Mon Sep 6 14:58:41 2021[1,0]: - ETA: 6:05:17 - loss: 0.1372 - auc: 0.7613Mon Sep 6 14:58:49 2021[1,0]:Mon Sep 6 14:58:41 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 14:59:42 2021[1,0]:1389/4000 [=========>....................]Mon Sep 6 14:59:42 2021[1,0]: - ETA: 6:04:23 - loss: 0.1371 - auc: 0.7614Mon Sep 6 14:59:50 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:00:43 2021[1,0]:1396/4000 [=========>....................]Mon Sep 6 15:00:43 2021[1,0]: - ETA: 6:03:29 - loss: 0.1370 - auc: 0.7615Mon Sep 6 15:00:51 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:01:43 2021[1,0]:1403/4000 [=========>....................]Mon Sep 6 15:01:43 2021[1,0]: - ETA: 6:02:33 - loss: 0.1370 - auc: 0.7616Mon Sep 6 15:01:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1410/4000 [=========>....................]Mon Sep 6 15:02:43 2021[1,0]: - ETA: 6:01:37 - loss: 0.1369 - auc: 0.7618Mon Sep 6 15:02:52 2021[1,0]:Mon Sep 6 15:02:43 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:03:43 2021[1,0]:1417/4000 [=========>....................]Mon Sep 6 15:03:43 2021[1,0]: - ETA: 6:00:40 - loss: 0.1369 - auc: 0.7619Mon Sep 6 15:03:51 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:04:43 2021[1,0]:1424/4000 [=========>....................]Mon Sep 6 15:04:43 2021[1,0]: - ETA: 5:59:44 - loss: 0.1368 - auc: 0.7620Mon Sep 6 15:04:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:05:43 2021[1,0]:1431/4000 [=========>....................]Mon Sep 6 15:05:43 2021[1,0]: - ETA: 5:58:48 - loss: 0.1368 - auc: 0.7621Mon Sep 6 15:05:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:06:44 2021[1,0]:1438/4000 [=========>....................]Mon Sep 6 15:06:44 2021[1,0]: - ETA: 5:57:53 - loss: 0.1367 - auc: 0.7622Mon Sep 6 15:06:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:07:45 2021[1,0]:1445/4000 [=========>....................]Mon Sep 6 15:07:45 2021[1,0]: - ETA: 5:56:58 - loss: 0.1367 - auc: 0.7623Mon Sep 6 15:07:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1452/4000 [=========>....................]Mon Sep 6 15:08:46 2021[1,0]: - ETA: 5:56:04 - loss: 0.1366 - auc: 0.7624Mon Sep 6 15:08:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:09:45 2021[1,0]:1459/4000 [=========>....................]Mon Sep 6 15:09:45 2021[1,0]: - ETA: 5:55:05 - loss: 0.1365 - auc: 0.7625Mon Sep 6 15:09:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:10:44 2021[1,0]:1466/4000 [=========>....................]Mon Sep 6 15:10:44 2021[1,0]: - ETA: 5:54:08 - loss: 0.1365 - auc: 0.7627Mon Sep 6 15:10:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1473/4000 [==========>...................]Mon Sep 6 15:11:42 2021[1,0]: - ETA: 5:53:08 - loss: 0.1364 - auc: 0.7627Mon Sep 6 15:11:51 2021[1,0]:Mon Sep 6 15:11:33 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:12:47 2021[1,0]:1481/4000 [==========>...................]Mon Sep 6 15:12:47 2021[1,0]: - ETA: 5:51:58 - loss: 0.1364 - auc: 0.7629Mon Sep 6 15:12:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1488/4000 [==========>...................]Mon Sep 6 15:13:44 2021[1,0]: - ETA: 5:50:57 - loss: 0.1363 - auc: 0.7630Mon Sep 6 15:13:52 2021[1,0]:Mon Sep 6 15:13:36 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:14:40 2021[1,0]:1495/4000 [==========>...................]Mon Sep 6 15:14:40 2021[1,0]: - ETA: 5:49:53 - loss: 0.1363 - auc: 0.7631Mon Sep 6 15:14:49 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:15:40 2021[1,0]:1502/4000 [==========>...................]Mon Sep 6 15:15:40 2021[1,0]: - ETA: 5:48:57 - loss: 0.1362 - auc: 0.7632Mon Sep 6 15:15:49 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:16:41 2021[1,0]:1509/4000 [==========>...................]Mon Sep 6 15:16:41 2021[1,0]: - ETA: 5:48:02 - loss: 0.1362 - auc: 0.7633Mon Sep 6 15:16:50 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:17:41 2021[1,0]:1516/4000 [==========>...................]Mon Sep 6 15:17:41 2021[1,0]: - ETA: 5:47:05 - loss: 0.1361 - auc: 0.7634Mon Sep 6 15:17:50 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1523/4000 [==========>...................]Mon Sep 6 15:18:40 2021[1,0]: - ETA: 5:46:08 - loss: 0.1361 - auc: 0.7635Mon Sep 6 15:18:49 2021[1,0]:Mon Sep 6 15:18:32 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:19:48 2021[1,0]:1531/4000 [==========>...................]Mon Sep 6 15:19:48 2021[1,0]: - ETA: 5:45:01 - loss: 0.1360 - auc: 0.7636Mon Sep 6 15:19:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:20:45 2021[1,0]:1538/4000 [==========>...................]Mon Sep 6 15:20:45 2021[1,0]: - ETA: 5:44:00 - loss: 0.1360 - auc: 0.7637Mon Sep 6 15:20:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:21:46 2021[1,0]:1545/4000 [==========>...................] - ETA: 5:43:05 - loss: 0.1359 - auc: 0.7638Mon Sep 6 15:21:54 2021[1,0]:Mon Sep 6 15:21:20 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1552/4000 [==========>...................]Mon Sep 6 15:22:46 2021[1,0]: - ETA: 5:42:08 - loss: 0.1359 - auc: 0.7640Mon Sep 6 15:22:54 2021[1,0]:Mon Sep 6 15:22:37 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:23:45 2021[1,0]:1559/4000 [==========>...................]Mon Sep 6 15:23:45 2021[1,0]: - ETA: 5:41:10 - loss: 0.1358 - auc: 0.7640Mon Sep 6 15:23:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:24:44 2021[1,0]:1566/4000 [==========>...................]Mon Sep 6 15:24:44 2021[1,0]: - ETA: 5:40:12 - loss: 0.1358 - auc: 0.7641Mon Sep 6 15:24:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:25:42 2021[1,0]:1573/4000 [==========>...................]Mon Sep 6 15:25:42 2021[1,0]: - ETA: 5:39:12 - loss: 0.1357 - auc: 0.7642Mon Sep 6 15:25:50 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:26:48 2021[1,0]:1581/4000 [==========>...................]Mon Sep 6 15:26:48 2021[1,0]: - ETA: 5:38:04 - loss: 0.1357 - auc: 0.7643Mon Sep 6 15:26:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:27:47 2021[1,0]:1588/4000 [==========>...................]Mon Sep 6 15:27:47 2021[1,0]: - ETA: 5:37:06 - loss: 0.1356 - auc: 0.7645Mon Sep 6 15:27:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:28:48 2021[1,0]:1595/4000 [==========>...................]Mon Sep 6 15:28:48 2021[1,0]: - ETA: 5:36:10 - loss: 0.1356 - auc: 0.7646Mon Sep 6 15:28:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:29:47 2021[1,0]:1602/4000 [===========>..................]Mon Sep 6 15:29:47 2021[1,0]: - ETA: 5:35:12 - loss: 0.1356 - auc: 0.7647Mon Sep 6 15:29:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1609/4000 [===========>..................]Mon Sep 6 15:30:47 2021[1,0]: - ETA: 5:34:15 - loss: 0.1355 - auc: 0.7648Mon Sep 6 15:30:55 2021[1,0]:Mon Sep 6 15:30:47 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:31:45 2021[1,0]:1616/4000 [===========>..................]Mon Sep 6 15:31:45 2021[1,0]: - ETA: 5:33:16 - loss: 0.1355 - auc: 0.7649Mon Sep 6 15:31:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:32:44 2021[1,0]:1623/4000 [===========>..................]Mon Sep 6 15:32:44 2021[1,0]: - ETA: 5:32:17 - loss: 0.1354 - auc: 0.7650Mon Sep 6 15:32:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:33:43 2021[1,0]:1630/4000 [===========>..................]Mon Sep 6 15:33:43 2021[1,0]: - ETA: 5:31:19 - loss: 0.1354 - auc: 0.7651Mon Sep 6 15:33:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:34:43 2021[1,0]:1637/4000 [===========>..................]Mon Sep 6 15:34:43 2021[1,0]: - ETA: 5:30:21 - loss: 0.1353 - auc: 0.7651Mon Sep 6 15:34:51 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:35:43 2021[1,0]:1644/4000 [===========>..................]Mon Sep 6 15:35:43 2021[1,0]: - ETA: 5:29:24 - loss: 0.1353 - auc: 0.7652Mon Sep 6 15:35:51 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:36:41 2021[1,0]:1651/4000 [===========>..................]Mon Sep 6 15:36:41 2021[1,0]: - ETA: 5:28:25 - loss: 0.1352 - auc: 0.7653Mon Sep 6 15:36:50 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:37:47 2021[1,0]:1659/4000 [===========>..................]Mon Sep 6 15:37:47 2021[1,0]: - ETA: 5:27:16 - loss: 0.1352 - auc: 0.7654Mon Sep 6 15:37:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:38:44 2021[1,0]:1666/4000 [===========>..................]Mon Sep 6 15:38:44 2021[1,0]: - ETA: 5:26:15 - loss: 0.1352 - auc: 0.7655Mon Sep 6 15:38:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:39:42 2021[1,0]:1673/4000 [===========>..................]Mon Sep 6 15:39:42 2021[1,0]: - ETA: 5:25:15 - loss: 0.1351 - auc: 0.7656Mon Sep 6 15:39:50 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:40:48 2021[1,0]:1681/4000 [===========>..................]Mon Sep 6 15:40:48 2021[1,0]: - ETA: 5:24:07 - loss: 0.1351 - auc: 0.7657Mon Sep 6 15:40:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:41:40 2021[1,0]:1687/4000 [===========>..................]Mon Sep 6 15:41:40 2021[1,0]: - ETA: 5:23:19 - loss: 0.1350 - auc: 0.7658Mon Sep 6 15:41:49 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:42:41 2021[1,0]:1694/4000 [===========>..................]Mon Sep 6 15:42:41 2021[1,0]: - ETA: 5:22:23 - loss: 0.1350 - auc: 0.7659Mon Sep 6 15:42:49 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:43:41 2021[1,0]:1701/4000 [===========>..................]Mon Sep 6 15:43:41 2021[1,0]: - ETA: 5:21:26 - loss: 0.1349 - auc: 0.7660Mon Sep 6 15:43:49 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1708/4000 [===========>..................]Mon Sep 6 15:44:42 2021[1,0]: - ETA: 5:20:30 - loss: 0.1349 - auc: 0.7661Mon Sep 6 15:44:50 2021[1,0]:Mon Sep 6 15:44:33 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1715/4000 [===========>..................]Mon Sep 6 15:45:42 2021[1,0]: - ETA: 5:19:34 - loss: 0.1349 - auc: 0.7661Mon Sep 6 15:45:51 2021[1,0]:Mon Sep 6 15:45:42 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:46:43 2021[1,0]:1722/4000 [===========>..................]Mon Sep 6 15:46:43 2021[1,0]: - ETA: 5:18:38 - loss: 0.1348 - auc: 0.7662Mon Sep 6 15:46:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:47:44 2021[1,0]:1729/4000 [===========>..................]Mon Sep 6 15:47:44 2021[1,0]: - ETA: 5:17:42 - loss: 0.1348 - auc: 0.7663Mon Sep 6 15:47:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:48:44 2021[1,0]:1736/4000 [============>.................]Mon Sep 6 15:48:44 2021[1,0]: - ETA: 5:16:45 - loss: 0.1348 - auc: 0.7664Mon Sep 6 15:48:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:49:41 2021[1,0]:1743/4000 [============>.................]Mon Sep 6 15:49:41 2021[1,0]: - ETA: 5:15:44 - loss: 0.1347 - auc: 0.7665Mon Sep 6 15:49:49 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:50:45 2021[1,0]:1751/4000 [============>.................]Mon Sep 6 15:50:45 2021[1,0]: - ETA: 5:14:33 - loss: 0.1347 - auc: 0.7666Mon Sep 6 15:50:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:51:47 2021[1,0]:1759/4000 [============>.................]Mon Sep 6 15:51:47 2021[1,0]: - ETA: 5:13:19 - loss: 0.1346 - auc: 0.7667Mon Sep 6 15:51:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:52:43 2021[1,0]:1766/4000 [============>.................]Mon Sep 6 15:52:43 2021[1,0]: - ETA: 5:12:16 - loss: 0.1346 - auc: 0.7667Mon Sep 6 15:52:50 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:53:49 2021[1,0]:1774/4000 [============>.................]Mon Sep 6 15:53:49 2021[1,0]: - ETA: 5:11:08 - loss: 0.1346 - auc: 0.7668Mon Sep 6 15:53:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:54:48 2021[1,0]:1781/4000 [============>.................]Mon Sep 6 15:54:48 2021[1,0]: - ETA: 5:10:10 - loss: 0.1345 - auc: 0.7669Mon Sep 6 15:54:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:55:47 2021[1,0]:1788/4000 [============>.................]Mon Sep 6 15:55:47 2021[1,0]: - ETA: 5:09:12 - loss: 0.1345 - auc: 0.7670Mon Sep 6 15:55:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:56:46 2021[1,0]:1795/4000 [============>.................]Mon Sep 6 15:56:46 2021[1,0]: - ETA: 5:08:13 - loss: 0.1344 - auc: 0.7671Mon Sep 6 15:56:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:57:44 2021[1,0]:1802/4000 [============>.................]Mon Sep 6 15:57:44 2021[1,0]: - ETA: 5:07:13 - loss: 0.1344 - auc: 0.7671Mon Sep 6 15:57:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:58:43 2021[1,0]:1809/4000 [============>.................]Mon Sep 6 15:58:43 2021[1,0]: - ETA: 5:06:15 - loss: 0.1344 - auc: 0.7672Mon Sep 6 15:58:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 15:59:42 2021[1,0]:1816/4000 [============>.................]Mon Sep 6 15:59:42 2021[1,0]: - ETA: 5:05:17 - loss: 0.1343 - auc: 0.7673Mon Sep 6 15:59:50 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:00:41 2021[1,0]:1823/4000 [============>.................]Mon Sep 6 16:00:41 2021[1,0]: - ETA: 5:04:19 - loss: 0.1343 - auc: 0.7674Mon Sep 6 16:00:50 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:01:41 2021[1,0]:1830/4000 [============>.................]Mon Sep 6 16:01:41 2021[1,0]: - ETA: 5:03:21 - loss: 0.1343 - auc: 0.7675Mon Sep 6 16:01:49 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1838/4000 [============>.................]Mon Sep 6 16:02:49 2021[1,0]: - ETA: 5:02:15 - loss: 0.1342 - auc: 0.7676Mon Sep 6 16:02:57 2021[1,0]:Mon Sep 6 16:02:40 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:03:41 2021[1,0]:1844/4000 [============>.................]Mon Sep 6 16:03:41 2021[1,0]: - ETA: 5:01:27 - loss: 0.1342 - auc: 0.7676Mon Sep 6 16:03:49 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:04:41 2021[1,0]:1851/4000 [============>.................]Mon Sep 6 16:04:41 2021[1,0]: - ETA: 5:00:30 - loss: 0.1342 - auc: 0.7677Mon Sep 6 16:04:50 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:05:41 2021[1,0]:1858/4000 [============>.................]Mon Sep 6 16:05:41 2021[1,0]: - ETA: 4:59:32 - loss: 0.1341 - auc: 0.7678Mon Sep 6 16:05:49 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:06:41 2021[1,0]:1865/4000 [============>.................] - ETA: 4:58:35 - loss: 0.1341 - auc: 0.7679Mon Sep 6 16:06:49 2021[1,0]:Mon Sep 6 16:06:32 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:07:41 2021[1,0]:1872/4000 [=============>................]Mon Sep 6 16:07:41 2021[1,0]: - ETA: 4:57:38 - loss: 0.1341 - auc: 0.7679Mon Sep 6 16:07:49 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:08:49 2021[1,0]:1880/4000 [=============>................]Mon Sep 6 16:08:49 2021[1,0]: - ETA: 4:56:31 - loss: 0.1340 - auc: 0.7680Mon Sep 6 16:08:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:09:47 2021[1,0]:1887/4000 [=============>................]Mon Sep 6 16:09:47 2021[1,0]: - ETA: 4:55:32 - loss: 0.1340 - auc: 0.7681Mon Sep 6 16:09:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:10:44 2021[1,0]:1894/4000 [=============>................]Mon Sep 6 16:10:44 2021[1,0]: - ETA: 4:54:31 - loss: 0.1340 - auc: 0.7682Mon Sep 6 16:10:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:11:43 2021[1,0]:1901/4000 [=============>................]Mon Sep 6 16:11:43 2021[1,0]: - ETA: 4:53:33 - loss: 0.1339 - auc: 0.7682Mon Sep 6 16:11:51 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:12:49 2021[1,0]:1909/4000 [=============>................]Mon Sep 6 16:12:49 2021[1,0]: - ETA: 4:52:24 - loss: 0.1339 - auc: 0.7683Mon Sep 6 16:12:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:13:45 2021[1,0]:1916/4000 [=============>................]Mon Sep 6 16:13:45 2021[1,0]: - ETA: 4:51:23 - loss: 0.1339 - auc: 0.7684Mon Sep 6 16:13:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:14:44 2021[1,0]:1923/4000 [=============>................]Mon Sep 6 16:14:44 2021[1,0]: - ETA: 4:50:24 - loss: 0.1338 - auc: 0.7685Mon Sep 6 16:14:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:15:48 2021[1,0]:1931/4000 [=============>................]Mon Sep 6 16:15:48 2021[1,0]: - ETA: 4:49:15 - loss: 0.1338 - auc: 0.7686Mon Sep 6 16:15:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:16:46 2021[1,0]:1938/4000 [=============>................]Mon Sep 6 16:16:46 2021[1,0]: - ETA: 4:48:14 - loss: 0.1338 - auc: 0.7686Mon Sep 6 16:16:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:17:43 2021[1,0]:1945/4000 [=============>................]Mon Sep 6 16:17:43 2021[1,0]: - ETA: 4:47:14 - loss: 0.1337 - auc: 0.7687Mon Sep 6 16:17:51 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:18:49 2021[1,0]:1953/4000 [=============>................]Mon Sep 6 16:18:49 2021[1,0]: - ETA: 4:46:06 - loss: 0.1337 - auc: 0.7688Mon Sep 6 16:18:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:19:48 2021[1,0]:1960/4000 [=============>................]Mon Sep 6 16:19:48 2021[1,0]: - ETA: 4:45:07 - loss: 0.1337 - auc: 0.7689Mon Sep 6 16:19:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:20:49 2021[1,0]:1967/4000 [=============>................]Mon Sep 6 16:20:49 2021[1,0]: - ETA: 4:44:11 - loss: 0.1336 - auc: 0.7689Mon Sep 6 16:20:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:21:41 2021[1,0]:1973/4000 [=============>................]Mon Sep 6 16:21:41 2021[1,0]: - ETA: 4:43:22 - loss: 0.1336 - auc: 0.7690Mon Sep 6 16:21:49 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:22:49 2021[1,0]:1981/4000 [=============>................]Mon Sep 6 16:22:49 2021[1,0]: - ETA: 4:42:16 - loss: 0.1336 - auc: 0.7691Mon Sep 6 16:22:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:23:45 2021[1,0]:1988/4000 [=============>................]Mon Sep 6 16:23:45 2021[1,0]: - ETA: 4:41:15 - loss: 0.1335 - auc: 0.7692Mon Sep 6 16:23:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:24:49 2021[1,0]:1996/4000 [=============>................]Mon Sep 6 16:24:49 2021[1,0]: - ETA: 4:40:04 - loss: 0.1335 - auc: 0.7692Mon Sep 6 16:24:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2003/4000 [==============>...............]Mon Sep 6 16:25:44 2021[1,0]: - ETA: 4:39:02 - loss: 0.1335 - auc: 0.7693Mon Sep 6 16:25:51 2021[1,0]:Mon Sep 6 16:25:44 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:26:45 2021[1,0]:2011/4000 [==============>...............]Mon Sep 6 16:26:45 2021[1,0]: - ETA: 4:37:49 - loss: 0.1334 - auc: 0.7694Mon Sep 6 16:26:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2018/4000 [==============>...............]Mon Sep 6 16:27:42 2021[1,0]: - ETA: 4:36:48 - loss: 0.1334 - auc: 0.7695Mon Sep 6 16:27:50 2021[1,0]:Mon Sep 6 16:27:25 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:28:49 2021[1,0]:2026/4000 [==============>...............]Mon Sep 6 16:28:49 2021[1,0]: - ETA: 4:35:42 - loss: 0.1334 - auc: 0.7695Mon Sep 6 16:28:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:29:49 2021[1,0]:2033/4000 [==============>...............]Mon Sep 6 16:29:49 2021[1,0]: - ETA: 4:34:45 - loss: 0.1333 - auc: 0.7696Mon Sep 6 16:29:58 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:30:49 2021[1,0]:2040/4000 [==============>...............] - ETA: 4:33:47 - loss: 0.1333 - auc: 0.7696Mon Sep 6 16:30:58 2021[1,0]:Mon Sep 6 16:30:41 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:31:49 2021[1,0]:2047/4000 [==============>...............]Mon Sep 6 16:31:49 2021[1,0]: - ETA: 4:32:49 - loss: 0.1333 - auc: 0.7697Mon Sep 6 16:31:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:32:47 2021[1,0]:2054/4000 [==============>...............]Mon Sep 6 16:32:47 2021[1,0]: - ETA: 4:31:51 - loss: 0.1333 - auc: 0.7697Mon Sep 6 16:32:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:33:47 2021[1,0]:2061/4000 [==============>...............]Mon Sep 6 16:33:47 2021[1,0]: - ETA: 4:30:52 - loss: 0.1332 - auc: 0.7698Mon Sep 6 16:33:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2068/4000 [==============>...............]Mon Sep 6 16:34:45 2021[1,0]: - ETA: 4:29:54 - loss: 0.1332 - auc: 0.7699Mon Sep 6 16:34:54 2021[1,0]:Mon Sep 6 16:34:45 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:35:46 2021[1,0]:2075/4000 [==============>...............] - ETA: 4:28:57 - loss: 0.1332 - auc: 0.7699Mon Sep 6 16:35:54 2021[1,0]:Mon Sep 6 16:35:20 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:36:44 2021[1,0]:2082/4000 [==============>...............]Mon Sep 6 16:36:44 2021[1,0]: - ETA: 4:27:57 - loss: 0.1331 - auc: 0.7700Mon Sep 6 16:36:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:37:47 2021[1,0]:2090/4000 [==============>...............]Mon Sep 6 16:37:47 2021[1,0]: - ETA: 4:26:47 - loss: 0.1331 - auc: 0.7700Mon Sep 6 16:37:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2097/4000 [==============>...............]Mon Sep 6 16:38:43 2021[1,0]: - ETA: 4:25:46 - loss: 0.1331 - auc: 0.7701Mon Sep 6 16:38:51 2021[1,0]:Mon Sep 6 16:38:43 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2105/4000 [==============>...............]Mon Sep 6 16:39:48 2021[1,0]: - ETA: 4:24:37 - loss: 0.1331 - auc: 0.7702Mon Sep 6 16:39:57 2021[1,0]:Mon Sep 6 16:39:48 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:40:48 2021[1,0]:2112/4000 [==============>...............]Mon Sep 6 16:40:48 2021[1,0]: - ETA: 4:23:39 - loss: 0.1330 - auc: 0.7702Mon Sep 6 16:40:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:41:48 2021[1,0]:2119/4000 [==============>...............] - ETA: 4:22:42 - loss: 0.1330 - auc: 0.7703Mon Sep 6 16:41:57 2021[1,0]:Mon Sep 6 16:41:39 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:42:48 2021[1,0]:2126/4000 [==============>...............]Mon Sep 6 16:42:48 2021[1,0]: - ETA: 4:21:45 - loss: 0.1330 - auc: 0.7703Mon Sep 6 16:42:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:43:48 2021[1,0]:2133/4000 [==============>...............]Mon Sep 6 16:43:48 2021[1,0]: - ETA: 4:20:47 - loss: 0.1330 - auc: 0.7704Mon Sep 6 16:43:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:44:48 2021[1,0]:2140/4000 [===============>..............]Mon Sep 6 16:44:48 2021[1,0]: - ETA: 4:19:50 - loss: 0.1329 - auc: 0.7704Mon Sep 6 16:44:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:45:47 2021[1,0]:2147/4000 [===============>..............]Mon Sep 6 16:45:47 2021[1,0]: - ETA: 4:18:51 - loss: 0.1329 - auc: 0.7705Mon Sep 6 16:45:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:46:47 2021[1,0]:2154/4000 [===============>..............]Mon Sep 6 16:46:47 2021[1,0]: - ETA: 4:17:54 - loss: 0.1329 - auc: 0.7705Mon Sep 6 16:46:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:47:48 2021[1,0]:2161/4000 [===============>..............]Mon Sep 6 16:47:48 2021[1,0]: - ETA: 4:16:57 - loss: 0.1328 - auc: 0.7706Mon Sep 6 16:47:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2168/4000 [===============>..............]Mon Sep 6 16:48:47 2021[1,0]: - ETA: 4:15:58 - loss: 0.1328 - auc: 0.7707Mon Sep 6 16:48:55 2021[1,0]:Mon Sep 6 16:48:47 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:49:44 2021[1,0]:2175/4000 [===============>..............]Mon Sep 6 16:49:44 2021[1,0]: - ETA: 4:14:59 - loss: 0.1328 - auc: 0.7707Mon Sep 6 16:49:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:50:43 2021[1,0]:2182/4000 [===============>..............]Mon Sep 6 16:50:43 2021[1,0]: - ETA: 4:14:00 - loss: 0.1328 - auc: 0.7708Mon Sep 6 16:50:51 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:51:49 2021[1,0]:2190/4000 [===============>..............]Mon Sep 6 16:51:49 2021[1,0]: - ETA: 4:12:52 - loss: 0.1327 - auc: 0.7708Mon Sep 6 16:51:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:52:48 2021[1,0]:2197/4000 [===============>..............]Mon Sep 6 16:52:48 2021[1,0]: - ETA: 4:11:53 - loss: 0.1327 - auc: 0.7709Mon Sep 6 16:52:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:53:47 2021[1,0]:2204/4000 [===============>..............]Mon Sep 6 16:53:47 2021[1,0]: - ETA: 4:10:55 - loss: 0.1327 - auc: 0.7710Mon Sep 6 16:53:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:54:47 2021[1,0]:2211/4000 [===============>..............]Mon Sep 6 16:54:47 2021[1,0]: - ETA: 4:09:58 - loss: 0.1327 - auc: 0.7710Mon Sep 6 16:54:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2218/4000 [===============>..............]Mon Sep 6 16:55:49 2021[1,0]: - ETA: 4:09:02 - loss: 0.1326 - auc: 0.7711Mon Sep 6 16:55:57 2021[1,0]:Mon Sep 6 16:55:40 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:56:48 2021[1,0]:2225/4000 [===============>..............]Mon Sep 6 16:56:48 2021[1,0]: - ETA: 4:08:03 - loss: 0.1326 - auc: 0.7711Mon Sep 6 16:56:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:57:46 2021[1,0]:2232/4000 [===============>..............]Mon Sep 6 16:57:46 2021[1,0]: - ETA: 4:07:04 - loss: 0.1326 - auc: 0.7712Mon Sep 6 16:57:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:58:44 2021[1,0]:2239/4000 [===============>..............]Mon Sep 6 16:58:44 2021[1,0]: - ETA: 4:06:05 - loss: 0.1326 - auc: 0.7712Mon Sep 6 16:58:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 16:59:42 2021[1,0]:2246/4000 [===============>..............]Mon Sep 6 16:59:42 2021[1,0]: - ETA: 4:05:06 - loss: 0.1326 - auc: 0.7713Mon Sep 6 16:59:51 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2254/4000 [===============>..............]Mon Sep 6 17:00:48 2021[1,0]: - ETA: 4:03:58 - loss: 0.1325 - auc: 0.7713Mon Sep 6 17:00:56 2021[1,0]:Mon Sep 6 17:00:48 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:01:47 2021[1,0]:2261/4000 [===============>..............]Mon Sep 6 17:01:47 2021[1,0]: - ETA: 4:02:59 - loss: 0.1325 - auc: 0.7714Mon Sep 6 17:01:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:02:45 2021[1,0]:2268/4000 [================>.............]Mon Sep 6 17:02:45 2021[1,0]: - ETA: 4:02:00 - loss: 0.1325 - auc: 0.7714Mon Sep 6 17:02:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:03:43 2021[1,0]:2275/4000 [================>.............]Mon Sep 6 17:03:43 2021[1,0]: - ETA: 4:01:01 - loss: 0.1325 - auc: 0.7715Mon Sep 6 17:03:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:04:43 2021[1,0]:2282/4000 [================>.............] - ETA: 4:00:03 - loss: 0.1324 - auc: 0.7715Mon Sep 6 17:04:51 2021[1,0]:Mon Sep 6 17:04:17 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:05:44 2021[1,0]:2289/4000 [================>.............]Mon Sep 6 17:05:44 2021[1,0]: - ETA: 3:59:06 - loss: 0.1324 - auc: 0.7716Mon Sep 6 17:05:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:06:45 2021[1,0]:2296/4000 [================>.............]Mon Sep 6 17:06:45 2021[1,0]: - ETA: 3:58:09 - loss: 0.1324 - auc: 0.7716Mon Sep 6 17:06:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:07:46 2021[1,0]:2303/4000 [================>.............]Mon Sep 6 17:07:46 2021[1,0]: - ETA: 3:57:12 - loss: 0.1324 - auc: 0.7717Mon Sep 6 17:07:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:08:46 2021[1,0]:2310/4000 [================>.............]Mon Sep 6 17:08:46 2021[1,0]: - ETA: 3:56:14 - loss: 0.1324 - auc: 0.7717Mon Sep 6 17:08:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:09:46 2021[1,0]:2317/4000 [================>.............]Mon Sep 6 17:09:46 2021[1,0]: - ETA: 3:55:16 - loss: 0.1323 - auc: 0.7718Mon Sep 6 17:09:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:10:44 2021[1,0]:2324/4000 [================>.............]Mon Sep 6 17:10:44 2021[1,0]: - ETA: 3:54:17 - loss: 0.1323 - auc: 0.7718Mon Sep 6 17:10:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:11:42 2021[1,0]:2331/4000 [================>.............]Mon Sep 6 17:11:42 2021[1,0]: - ETA: 3:53:18 - loss: 0.1323 - auc: 0.7719Mon Sep 6 17:11:51 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:12:48 2021[1,0]:2339/4000 [================>.............]Mon Sep 6 17:12:48 2021[1,0]: - ETA: 3:52:11 - loss: 0.1323 - auc: 0.7719Mon Sep 6 17:12:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:13:48 2021[1,0]:2346/4000 [================>.............]Mon Sep 6 17:13:48 2021[1,0]: - ETA: 3:51:12 - loss: 0.1322 - auc: 0.7720Mon Sep 6 17:13:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2353/4000 [================>.............]Mon Sep 6 17:14:49 2021[1,0]: - ETA: 3:50:15 - loss: 0.1322 - auc: 0.7720Mon Sep 6 17:14:57 2021[1,0]:Mon Sep 6 17:14:49 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:15:48 2021[1,0]:2360/4000 [================>.............]Mon Sep 6 17:15:48 2021[1,0]: - ETA: 3:49:17 - loss: 0.1322 - auc: 0.7721Mon Sep 6 17:15:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:16:48 2021[1,0]:2367/4000 [================>.............]Mon Sep 6 17:16:48 2021[1,0]: - ETA: 3:48:19 - loss: 0.1322 - auc: 0.7721Mon Sep 6 17:16:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2374/4000 [================>.............]Mon Sep 6 17:17:48 2021[1,0]: - ETA: 3:47:21 - loss: 0.1322 - auc: 0.7722Mon Sep 6 17:17:56 2021[1,0]:Mon Sep 6 17:17:30 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:18:46 2021[1,0]:2381/4000 [================>.............]Mon Sep 6 17:18:46 2021[1,0]: - ETA: 3:46:22 - loss: 0.1321 - auc: 0.7722Mon Sep 6 17:18:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:19:44 2021[1,0]:2388/4000 [================>.............]Mon Sep 6 17:19:44 2021[1,0]: - ETA: 3:45:22 - loss: 0.1321 - auc: 0.7723Mon Sep 6 17:19:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:20:49 2021[1,0]:2396/4000 [================>.............]Mon Sep 6 17:20:49 2021[1,0]: - ETA: 3:44:14 - loss: 0.1321 - auc: 0.7723Mon Sep 6 17:20:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:21:46 2021[1,0]:2403/4000 [=================>............]Mon Sep 6 17:21:46 2021[1,0]: - ETA: 3:43:14 - loss: 0.1321 - auc: 0.7724Mon Sep 6 17:21:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:22:43 2021[1,0]:2410/4000 [=================>............]Mon Sep 6 17:22:43 2021[1,0]: - ETA: 3:42:14 - loss: 0.1320 - auc: 0.7724Mon Sep 6 17:22:51 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:23:50 2021[1,0]:2418/4000 [=================>............]Mon Sep 6 17:23:50 2021[1,0]: - ETA: 3:41:07 - loss: 0.1320 - auc: 0.7725Mon Sep 6 17:23:58 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:24:49 2021[1,0]:2425/4000 [=================>............]Mon Sep 6 17:24:49 2021[1,0]: - ETA: 3:40:09 - loss: 0.1320 - auc: 0.7725Mon Sep 6 17:24:58 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:25:49 2021[1,0]:2432/4000 [=================>............]Mon Sep 6 17:25:49 2021[1,0]: - ETA: 3:39:11 - loss: 0.1320 - auc: 0.7726Mon Sep 6 17:25:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:26:46 2021[1,0]:2439/4000 [=================>............]Mon Sep 6 17:26:46 2021[1,0]: - ETA: 3:38:11 - loss: 0.1320 - auc: 0.7726Mon Sep 6 17:26:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:27:50 2021[1,0]:2447/4000 [=================>............]Mon Sep 6 17:27:50 2021[1,0]: - ETA: 3:37:02 - loss: 0.1319 - auc: 0.7727Mon Sep 6 17:27:58 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2454/4000 [=================>............]Mon Sep 6 17:28:45 2021[1,0]: - ETA: 3:36:01 - loss: 0.1319 - auc: 0.7727Mon Sep 6 17:28:53 2021[1,0]:Mon Sep 6 17:28:45 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:29:51 2021[1,0]:2462/4000 [=================>............]Mon Sep 6 17:29:51 2021[1,0]: - ETA: 3:34:53 - loss: 0.1319 - auc: 0.7728Mon Sep 6 17:29:59 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:30:49 2021[1,0]:2469/4000 [=================>............]Mon Sep 6 17:30:49 2021[1,0]: - ETA: 3:33:54 - loss: 0.1319 - auc: 0.7728Mon Sep 6 17:30:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:31:48 2021[1,0]:2476/4000 [=================>............]Mon Sep 6 17:31:48 2021[1,0]: - ETA: 3:32:56 - loss: 0.1318 - auc: 0.7729Mon Sep 6 17:31:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:32:48 2021[1,0]:2483/4000 [=================>............]Mon Sep 6 17:32:48 2021[1,0]: - ETA: 3:31:58 - loss: 0.1318 - auc: 0.7729Mon Sep 6 17:32:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:33:48 2021[1,0]:2490/4000 [=================>............]Mon Sep 6 17:33:48 2021[1,0]: - ETA: 3:31:00 - loss: 0.1318 - auc: 0.7730Mon Sep 6 17:33:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:34:48 2021[1,0]:2497/4000 [=================>............]Mon Sep 6 17:34:48 2021[1,0]: - ETA: 3:30:02 - loss: 0.1318 - auc: 0.7730Mon Sep 6 17:34:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:35:48 2021[1,0]:2504/4000 [=================>............]Mon Sep 6 17:35:48 2021[1,0]: - ETA: 3:29:04 - loss: 0.1318 - auc: 0.7730Mon Sep 6 17:35:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:36:44 2021[1,0]:2511/4000 [=================>............]Mon Sep 6 17:36:44 2021[1,0]: - ETA: 3:28:04 - loss: 0.1317 - auc: 0.7731Mon Sep 6 17:36:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2519/4000 [=================>............]Mon Sep 6 17:37:47 2021[1,0]: - ETA: 3:26:54 - loss: 0.1317 - auc: 0.7731Mon Sep 6 17:37:55 2021[1,0]:Mon Sep 6 17:37:24 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:38:44 2021[1,0]:2526/4000 [=================>............]Mon Sep 6 17:38:44 2021[1,0]: - ETA: 3:25:55 - loss: 0.1317 - auc: 0.7732Mon Sep 6 17:38:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:39:44 2021[1,0]:2533/4000 [=================>............]Mon Sep 6 17:39:44 2021[1,0]: - ETA: 3:24:57 - loss: 0.1317 - auc: 0.7733Mon Sep 6 17:39:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2540/4000 [==================>...........]Mon Sep 6 17:40:44 2021[1,0]: - ETA: 3:23:59 - loss: 0.1317 - auc: 0.7733Mon Sep 6 17:40:53 2021[1,0]:Mon Sep 6 17:40:18 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2547/4000 [==================>...........]Mon Sep 6 17:41:44 2021[1,0]: - ETA: 3:23:01 - loss: 0.1316 - auc: 0.7733Mon Sep 6 17:41:53 2021[1,0]:Mon Sep 6 17:41:44 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2554/4000 [==================>...........]Mon Sep 6 17:42:43 2021[1,0]: - ETA: 3:22:03 - loss: 0.1316 - auc: 0.7734Mon Sep 6 17:42:52 2021[1,0]:Mon Sep 6 17:42:43 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:43:42 2021[1,0]:2561/4000 [==================>...........]Mon Sep 6 17:43:42 2021[1,0]: - ETA: 3:21:04 - loss: 0.1316 - auc: 0.7734Mon Sep 6 17:43:51 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:44:50 2021[1,0]:2569/4000 [==================>...........]Mon Sep 6 17:44:50 2021[1,0]: - ETA: 3:19:57 - loss: 0.1316 - auc: 0.7735Mon Sep 6 17:44:59 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:45:49 2021[1,0]:2576/4000 [==================>...........]Mon Sep 6 17:45:49 2021[1,0]: - ETA: 3:18:59 - loss: 0.1316 - auc: 0.7735Mon Sep 6 17:45:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:46:47 2021[1,0]:2583/4000 [==================>...........]Mon Sep 6 17:46:47 2021[1,0]: - ETA: 3:18:00 - loss: 0.1315 - auc: 0.7735Mon Sep 6 17:46:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:47:46 2021[1,0]:2590/4000 [==================>...........]Mon Sep 6 17:47:46 2021[1,0]: - ETA: 3:17:01 - loss: 0.1315 - auc: 0.7736Mon Sep 6 17:47:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:48:44 2021[1,0]:2597/4000 [==================>...........]Mon Sep 6 17:48:44 2021[1,0]: - ETA: 3:16:02 - loss: 0.1315 - auc: 0.7736Mon Sep 6 17:48:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:49:43 2021[1,0]:2604/4000 [==================>...........]Mon Sep 6 17:49:43 2021[1,0]: - ETA: 3:15:03 - loss: 0.1315 - auc: 0.7737Mon Sep 6 17:49:51 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:50:50 2021[1,0]:2612/4000 [==================>...........]Mon Sep 6 17:50:50 2021[1,0]: - ETA: 3:13:57 - loss: 0.1315 - auc: 0.7737Mon Sep 6 17:50:59 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:51:49 2021[1,0]:2619/4000 [==================>...........]Mon Sep 6 17:51:49 2021[1,0]: - ETA: 3:12:58 - loss: 0.1314 - auc: 0.7738Mon Sep 6 17:51:58 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:52:49 2021[1,0]:2626/4000 [==================>...........]Mon Sep 6 17:52:49 2021[1,0]: - ETA: 3:12:00 - loss: 0.1314 - auc: 0.7738Mon Sep 6 17:52:58 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:53:48 2021[1,0]:2633/4000 [==================>...........]Mon Sep 6 17:53:48 2021[1,0]: - ETA: 3:11:01 - loss: 0.1314 - auc: 0.7739Mon Sep 6 17:53:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:54:47 2021[1,0]:2640/4000 [==================>...........]Mon Sep 6 17:54:47 2021[1,0]: - ETA: 3:10:03 - loss: 0.1314 - auc: 0.7739Mon Sep 6 17:54:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2647/4000 [==================>...........]Mon Sep 6 17:55:47 2021[1,0]: - ETA: 3:09:05 - loss: 0.1314 - auc: 0.7740Mon Sep 6 17:55:56 2021[1,0]:Mon Sep 6 17:55:30 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:56:47 2021[1,0]:2654/4000 [==================>...........]Mon Sep 6 17:56:47 2021[1,0]: - ETA: 3:08:07 - loss: 0.1313 - auc: 0.7740Mon Sep 6 17:56:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:57:46 2021[1,0]:2661/4000 [==================>...........]Mon Sep 6 17:57:46 2021[1,0]: - ETA: 3:07:08 - loss: 0.1313 - auc: 0.7740Mon Sep 6 17:57:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:58:46 2021[1,0]:2668/4000 [===================>..........]Mon Sep 6 17:58:46 2021[1,0]: - ETA: 3:06:10 - loss: 0.1313 - auc: 0.7741Mon Sep 6 17:58:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 17:59:51 2021[1,0]:2676/4000 [===================>..........]Mon Sep 6 17:59:51 2021[1,0]: - ETA: 3:05:02 - loss: 0.1313 - auc: 0.7741Mon Sep 6 17:59:59 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:00:49 2021[1,0]:2683/4000 [===================>..........]Mon Sep 6 18:00:49 2021[1,0]: - ETA: 3:04:03 - loss: 0.1313 - auc: 0.7742Mon Sep 6 18:00:58 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:01:49 2021[1,0]:2690/4000 [===================>..........]Mon Sep 6 18:01:49 2021[1,0]: - ETA: 3:03:05 - loss: 0.1313 - auc: 0.7742Mon Sep 6 18:01:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2697/4000 [===================>..........]Mon Sep 6 18:02:48 2021[1,0]: - ETA: 3:02:06 - loss: 0.1313 - auc: 0.7742Mon Sep 6 18:02:56 2021[1,0]:Mon Sep 6 18:02:48 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:03:48 2021[1,0]:2704/4000 [===================>..........]Mon Sep 6 18:03:48 2021[1,0]: - ETA: 3:01:09 - loss: 0.1312 - auc: 0.7743Mon Sep 6 18:03:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2711/4000 [===================>..........]Mon Sep 6 18:04:48 2021[1,0]: - ETA: 3:00:10 - loss: 0.1312 - auc: 0.7743Mon Sep 6 18:04:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:05:47 2021[1,0]:2718/4000 [===================>..........]Mon Sep 6 18:05:47 2021[1,0]: - ETA: 2:59:12 - loss: 0.1312 - auc: 0.7744Mon Sep 6 18:05:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:06:45 2021[1,0]:2725/4000 [===================>..........]Mon Sep 6 18:06:45 2021[1,0]: - ETA: 2:58:13 - loss: 0.1312 - auc: 0.7744Mon Sep 6 18:06:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:07:51 2021[1,0]:2733/4000 [===================>..........]Mon Sep 6 18:07:51 2021[1,0]: - ETA: 2:57:05 - loss: 0.1312 - auc: 0.7745Mon Sep 6 18:07:59 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:08:48 2021[1,0]:2740/4000 [===================>..........]Mon Sep 6 18:08:48 2021[1,0]: - ETA: 2:56:06 - loss: 0.1312 - auc: 0.7745Mon Sep 6 18:08:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:09:46 2021[1,0]:2747/4000 [===================>..........]Mon Sep 6 18:09:46 2021[1,0]: - ETA: 2:55:07 - loss: 0.1311 - auc: 0.7745Mon Sep 6 18:09:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2754/4000 [===================>..........]Mon Sep 6 18:10:44 2021[1,0]: - ETA: 2:54:08 - loss: 0.1311 - auc: 0.7746Mon Sep 6 18:10:52 2021[1,0]:Mon Sep 6 18:10:28 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:11:50 2021[1,0]:2762/4000 [===================>..........]Mon Sep 6 18:11:50 2021[1,0]: - ETA: 2:53:00 - loss: 0.1311 - auc: 0.7746Mon Sep 6 18:11:58 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:12:49 2021[1,0]:2769/4000 [===================>..........]Mon Sep 6 18:12:49 2021[1,0]: - ETA: 2:52:01 - loss: 0.1311 - auc: 0.7747Mon Sep 6 18:12:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:13:48 2021[1,0]:2776/4000 [===================>..........]Mon Sep 6 18:13:48 2021[1,0]: - ETA: 2:51:03 - loss: 0.1311 - auc: 0.7747Mon Sep 6 18:13:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:14:47 2021[1,0]:2783/4000 [===================>..........]Mon Sep 6 18:14:47 2021[1,0]: - ETA: 2:50:04 - loss: 0.1310 - auc: 0.7748Mon Sep 6 18:14:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:15:47 2021[1,0]:2790/4000 [===================>..........]Mon Sep 6 18:15:47 2021[1,0]: - ETA: 2:49:06 - loss: 0.1310 - auc: 0.7748Mon Sep 6 18:15:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:16:48 2021[1,0]:2797/4000 [===================>..........]Mon Sep 6 18:16:48 2021[1,0]: - ETA: 2:48:08 - loss: 0.1310 - auc: 0.7748Mon Sep 6 18:16:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2804/4000 [====================>.........]Mon Sep 6 18:17:47 2021[1,0]: - ETA: 2:47:10 - loss: 0.1310 - auc: 0.7749Mon Sep 6 18:17:56 2021[1,0]:Mon Sep 6 18:17:47 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:18:46 2021[1,0]:2811/4000 [====================>.........]Mon Sep 6 18:18:46 2021[1,0]: - ETA: 2:46:11 - loss: 0.1310 - auc: 0.7749Mon Sep 6 18:18:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2818/4000 [====================>.........]Mon Sep 6 18:19:47 2021[1,0]: - ETA: 2:45:14 - loss: 0.1310 - auc: 0.7749Mon Sep 6 18:19:56 2021[1,0]:Mon Sep 6 18:19:38 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:20:47 2021[1,0]:2825/4000 [====================>.........]Mon Sep 6 18:20:47 2021[1,0]: - ETA: 2:44:15 - loss: 0.1310 - auc: 0.7750Mon Sep 6 18:20:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:21:46 2021[1,0]:2832/4000 [====================>.........]Mon Sep 6 18:21:46 2021[1,0]: - ETA: 2:43:17 - loss: 0.1309 - auc: 0.7750Mon Sep 6 18:21:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2840/4000 [====================>.........]Mon Sep 6 18:22:52 2021[1,0]: - ETA: 2:42:09 - loss: 0.1309 - auc: 0.7751Mon Sep 6 18:23:00 2021[1,0]:Mon Sep 6 18:22:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2847/4000 [====================>.........]Mon Sep 6 18:23:51 2021[1,0]: - ETA: 2:41:11 - loss: 0.1309 - auc: 0.7751Mon Sep 6 18:24:00 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:24:44 2021[1,0]:2853/4000 [====================>.........]Mon Sep 6 18:24:44 2021[1,0]: - ETA: 2:40:21 - loss: 0.1309 - auc: 0.7751Mon Sep 6 18:24:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:25:45 2021[1,0]:2860/4000 [====================>.........]Mon Sep 6 18:25:45 2021[1,0]: - ETA: 2:39:23 - loss: 0.1309 - auc: 0.7752Mon Sep 6 18:25:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:26:45 2021[1,0]:2867/4000 [====================>.........]Mon Sep 6 18:26:45 2021[1,0]: - ETA: 2:38:25 - loss: 0.1309 - auc: 0.7752Mon Sep 6 18:26:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:27:45 2021[1,0]:2874/4000 [====================>.........]Mon Sep 6 18:27:45 2021[1,0]: - ETA: 2:37:27 - loss: 0.1308 - auc: 0.7752Mon Sep 6 18:27:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:28:45 2021[1,0]:2881/4000 [====================>.........]Mon Sep 6 18:28:45 2021[1,0]: - ETA: 2:36:29 - loss: 0.1308 - auc: 0.7753Mon Sep 6 18:28:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:29:44 2021[1,0]:2888/4000 [====================>.........]Mon Sep 6 18:29:44 2021[1,0]: - ETA: 2:35:30 - loss: 0.1308 - auc: 0.7753Mon Sep 6 18:29:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:30:52 2021[1,0]:2896/4000 [====================>.........]Mon Sep 6 18:30:52 2021[1,0]: - ETA: 2:34:23 - loss: 0.1308 - auc: 0.7754Mon Sep 6 18:31:00 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:31:50 2021[1,0]:2903/4000 [====================>.........]Mon Sep 6 18:31:50 2021[1,0]: - ETA: 2:33:24 - loss: 0.1308 - auc: 0.7754Mon Sep 6 18:31:58 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2910/4000 [====================>.........]Mon Sep 6 18:32:48 2021[1,0]: - ETA: 2:32:25 - loss: 0.1308 - auc: 0.7754Mon Sep 6 18:32:56 2021[1,0]:Mon Sep 6 18:32:48 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:33:47 2021[1,0]:2917/4000 [====================>.........]Mon Sep 6 18:33:47 2021[1,0]: - ETA: 2:31:27 - loss: 0.1308 - auc: 0.7755Mon Sep 6 18:33:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:34:45 2021[1,0]:2924/4000 [====================>.........]Mon Sep 6 18:34:45 2021[1,0]: - ETA: 2:30:28 - loss: 0.1307 - auc: 0.7755Mon Sep 6 18:34:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:35:45 2021[1,0]:2931/4000 [====================>.........]Mon Sep 6 18:35:45 2021[1,0]: - ETA: 2:29:30 - loss: 0.1307 - auc: 0.7755Mon Sep 6 18:35:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2938/4000 [=====================>........]Mon Sep 6 18:36:46 2021[1,0]: - ETA: 2:28:32 - loss: 0.1307 - auc: 0.7756Mon Sep 6 18:36:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:37:45 2021[1,0]:2945/4000 [=====================>........]Mon Sep 6 18:37:45 2021[1,0]: - ETA: 2:27:33 - loss: 0.1307 - auc: 0.7756Mon Sep 6 18:37:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:38:46 2021[1,0]:2952/4000 [=====================>........]Mon Sep 6 18:38:46 2021[1,0]: - ETA: 2:26:35 - loss: 0.1307 - auc: 0.7756Mon Sep 6 18:38:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:39:45 2021[1,0]:2959/4000 [=====================>........]Mon Sep 6 18:39:45 2021[1,0]: - ETA: 2:25:36 - loss: 0.1307 - auc: 0.7757Mon Sep 6 18:39:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2966/4000 [=====================>........]Mon Sep 6 18:40:45 2021[1,0]: - ETA: 2:24:38 - loss: 0.1307 - auc: 0.7757Mon Sep 6 18:40:54 2021[1,0]:Mon Sep 6 18:40:36 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:41:45 2021[1,0]:2973/4000 [=====================>........] - ETA: 2:23:40 - loss: 0.1306 - auc: 0.7757Mon Sep 6 18:41:54 2021[1,0]:Mon Sep 6 18:41:28 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:42:44 2021[1,0]:2980/4000 [=====================>........]Mon Sep 6 18:42:44 2021[1,0]: - ETA: 2:22:41 - loss: 0.1306 - auc: 0.7758Mon Sep 6 18:42:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:43:51 2021[1,0]:2988/4000 [=====================>........]Mon Sep 6 18:43:51 2021[1,0]: - ETA: 2:21:34 - loss: 0.1306 - auc: 0.7758Mon Sep 6 18:43:59 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2995/4000 [=====================>........]Mon Sep 6 18:44:49 2021[1,0]: - ETA: 2:20:35 - loss: 0.1306 - auc: 0.7758Mon Sep 6 18:44:58 2021[1,0]:Mon Sep 6 18:44:49 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:45:48 2021[1,0]:3002/4000 [=====================>........]Mon Sep 6 18:45:48 2021[1,0]: - ETA: 2:19:36 - loss: 0.1306 - auc: 0.7759Mon Sep 6 18:45:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:46:48 2021[1,0]:3009/4000 [=====================>........]Mon Sep 6 18:46:48 2021[1,0]: - ETA: 2:18:38 - loss: 0.1306 - auc: 0.7759Mon Sep 6 18:46:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:47:48 2021[1,0]:3016/4000 [=====================>........]Mon Sep 6 18:47:48 2021[1,0]: - ETA: 2:17:39 - loss: 0.1306 - auc: 0.7759Mon Sep 6 18:47:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3023/4000 [=====================>........]Mon Sep 6 18:48:48 2021[1,0]: - ETA: 2:16:41 - loss: 0.1305 - auc: 0.7759Mon Sep 6 18:48:56 2021[1,0]:Mon Sep 6 18:48:30 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:49:46 2021[1,0]:3030/4000 [=====================>........]Mon Sep 6 18:49:46 2021[1,0]: - ETA: 2:15:42 - loss: 0.1305 - auc: 0.7760Mon Sep 6 18:49:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:50:45 2021[1,0]:3037/4000 [=====================>........]Mon Sep 6 18:50:45 2021[1,0]: - ETA: 2:14:43 - loss: 0.1305 - auc: 0.7760Mon Sep 6 18:50:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3044/4000 [=====================>........]Mon Sep 6 18:51:45 2021[1,0]: - ETA: 2:13:45 - loss: 0.1305 - auc: 0.7760Mon Sep 6 18:51:53 2021[1,0]:Mon Sep 6 18:51:45 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:52:45 2021[1,0]:3051/4000 [=====================>........]Mon Sep 6 18:52:45 2021[1,0]: - ETA: 2:12:47 - loss: 0.1305 - auc: 0.7761Mon Sep 6 18:52:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:53:45 2021[1,0]:3058/4000 [=====================>........]Mon Sep 6 18:53:45 2021[1,0]: - ETA: 2:11:48 - loss: 0.1305 - auc: 0.7761Mon Sep 6 18:53:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:54:46 2021[1,0]:3065/4000 [=====================>........]Mon Sep 6 18:54:46 2021[1,0]: - ETA: 2:10:50 - loss: 0.1305 - auc: 0.7761Mon Sep 6 18:54:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:55:46 2021[1,0]:3072/4000 [======================>.......]Mon Sep 6 18:55:46 2021[1,0]: - ETA: 2:09:52 - loss: 0.1305 - auc: 0.7762Mon Sep 6 18:55:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:56:45 2021[1,0]:3079/4000 [======================>.......]Mon Sep 6 18:56:45 2021[1,0]: - ETA: 2:08:53 - loss: 0.1304 - auc: 0.7762Mon Sep 6 18:56:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:57:45 2021[1,0]:3086/4000 [======================>.......]Mon Sep 6 18:57:45 2021[1,0]: - ETA: 2:07:55 - loss: 0.1304 - auc: 0.7762Mon Sep 6 18:57:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:58:46 2021[1,0]:3093/4000 [======================>.......]Mon Sep 6 18:58:46 2021[1,0]: - ETA: 2:06:56 - loss: 0.1304 - auc: 0.7763Mon Sep 6 18:58:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 18:59:46 2021[1,0]:3100/4000 [======================>.......]Mon Sep 6 18:59:46 2021[1,0]: - ETA: 2:05:58 - loss: 0.1304 - auc: 0.7763Mon Sep 6 18:59:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3107/4000 [======================>.......]Mon Sep 6 19:00:47 2021[1,0]: - ETA: 2:05:00 - loss: 0.1304 - auc: 0.7763Mon Sep 6 19:00:56 2021[1,0]:Mon Sep 6 19:00:39 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:01:49 2021[1,0]:3114/4000 [======================>.......]Mon Sep 6 19:01:49 2021[1,0]: - ETA: 2:04:02 - loss: 0.1304 - auc: 0.7763Mon Sep 6 19:01:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:02:50 2021[1,0]:3121/4000 [======================>.......]Mon Sep 6 19:02:50 2021[1,0]: - ETA: 2:03:04 - loss: 0.1304 - auc: 0.7764Mon Sep 6 19:02:58 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:03:51 2021[1,0]:3128/4000 [======================>.......]Mon Sep 6 19:03:51 2021[1,0]: - ETA: 2:02:06 - loss: 0.1304 - auc: 0.7764Mon Sep 6 19:04:00 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:04:52 2021[1,0]:3135/4000 [======================>.......]Mon Sep 6 19:04:52 2021[1,0]: - ETA: 2:01:07 - loss: 0.1303 - auc: 0.7764Mon Sep 6 19:05:00 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:05:44 2021[1,0]:3141/4000 [======================>.......]Mon Sep 6 19:05:44 2021[1,0]: - ETA: 2:00:17 - loss: 0.1303 - auc: 0.7764Mon Sep 6 19:05:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:06:45 2021[1,0]:3148/4000 [======================>.......]Mon Sep 6 19:06:45 2021[1,0]: - ETA: 1:59:19 - loss: 0.1303 - auc: 0.7765Mon Sep 6 19:06:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:07:46 2021[1,0]:3155/4000 [======================>.......]Mon Sep 6 19:07:46 2021[1,0]: - ETA: 1:58:21 - loss: 0.1303 - auc: 0.7765Mon Sep 6 19:07:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:08:47 2021[1,0]:3162/4000 [======================>.......]Mon Sep 6 19:08:47 2021[1,0]: - ETA: 1:57:23 - loss: 0.1303 - auc: 0.7765Mon Sep 6 19:08:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3169/4000 [======================>.......]Mon Sep 6 19:09:49 2021[1,0]: - ETA: 1:56:25 - loss: 0.1303 - auc: 0.7766Mon Sep 6 19:09:57 2021[1,0]:Mon Sep 6 19:09:49 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:10:49 2021[1,0]:3176/4000 [======================>.......]Mon Sep 6 19:10:49 2021[1,0]: - ETA: 1:55:26 - loss: 0.1303 - auc: 0.7766Mon Sep 6 19:10:58 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:11:51 2021[1,0]:3183/4000 [======================>.......]Mon Sep 6 19:11:51 2021[1,0]: - ETA: 1:54:28 - loss: 0.1303 - auc: 0.7766Mon Sep 6 19:12:00 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:12:51 2021[1,0]:3190/4000 [======================>.......] - ETA: 1:53:30 - loss: 0.1303 - auc: 0.7766Mon Sep 6 19:13:00 2021[1,0]:Mon Sep 6 19:12:43 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:13:52 2021[1,0]:3197/4000 [======================>.......]Mon Sep 6 19:13:52 2021[1,0]: - ETA: 1:52:31 - loss: 0.1302 - auc: 0.7767Mon Sep 6 19:14:01 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:14:52 2021[1,0]:3204/4000 [=======================>......]Mon Sep 6 19:14:52 2021[1,0]: - ETA: 1:51:33 - loss: 0.1302 - auc: 0.7767Mon Sep 6 19:15:01 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:15:44 2021[1,0]:3210/4000 [=======================>......]Mon Sep 6 19:15:44 2021[1,0]: - ETA: 1:50:43 - loss: 0.1302 - auc: 0.7767Mon Sep 6 19:15:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:16:45 2021[1,0]:3217/4000 [=======================>......]Mon Sep 6 19:16:45 2021[1,0]: - ETA: 1:49:44 - loss: 0.1302 - auc: 0.7767Mon Sep 6 19:16:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3224/4000 [=======================>......]Mon Sep 6 19:17:45 2021[1,0]: - ETA: 1:48:46 - loss: 0.1302 - auc: 0.7768Mon Sep 6 19:17:53 2021[1,0]:Mon Sep 6 19:17:36 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:18:44 2021[1,0]:3231/4000 [=======================>......]Mon Sep 6 19:18:44 2021[1,0]: - ETA: 1:47:47 - loss: 0.1302 - auc: 0.7768Mon Sep 6 19:18:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:19:53 2021[1,0]:3239/4000 [=======================>......]Mon Sep 6 19:19:53 2021[1,0]: - ETA: 1:46:40 - loss: 0.1302 - auc: 0.7768Mon Sep 6 19:20:01 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3245/4000 [=======================>......]Mon Sep 6 19:20:45 2021[1,0]: - ETA: 1:45:50 - loss: 0.1302 - auc: 0.7768Mon Sep 6 19:20:53 2021[1,0]:Mon Sep 6 19:20:36 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:21:46 2021[1,0]:3252/4000 [=======================>......]Mon Sep 6 19:21:46 2021[1,0]: - ETA: 1:44:51 - loss: 0.1302 - auc: 0.7769Mon Sep 6 19:21:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:22:45 2021[1,0]:3259/4000 [=======================>......]Mon Sep 6 19:22:45 2021[1,0]: - ETA: 1:43:53 - loss: 0.1301 - auc: 0.7769Mon Sep 6 19:22:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:23:53 2021[1,0]:3267/4000 [=======================>......]Mon Sep 6 19:23:53 2021[1,0]: - ETA: 1:42:45 - loss: 0.1301 - auc: 0.7769Mon Sep 6 19:24:01 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:24:52 2021[1,0]:3274/4000 [=======================>......]Mon Sep 6 19:24:52 2021[1,0]: - ETA: 1:41:47 - loss: 0.1301 - auc: 0.7770Mon Sep 6 19:25:00 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3281/4000 [=======================>......]Mon Sep 6 19:25:51 2021[1,0]: - ETA: 1:40:48 - loss: 0.1301 - auc: 0.7770Mon Sep 6 19:25:59 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:26:50 2021[1,0]:3288/4000 [=======================>......]Mon Sep 6 19:26:50 2021[1,0]: - ETA: 1:39:49 - loss: 0.1301 - auc: 0.7770Mon Sep 6 19:26:58 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:27:49 2021[1,0]:3295/4000 [=======================>......]Mon Sep 6 19:27:49 2021[1,0]: - ETA: 1:38:50 - loss: 0.1301 - auc: 0.7770Mon Sep 6 19:27:58 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:28:48 2021[1,0]:3302/4000 [=======================>......]Mon Sep 6 19:28:48 2021[1,0]: - ETA: 1:37:51 - loss: 0.1301 - auc: 0.7771Mon Sep 6 19:28:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:29:47 2021[1,0]:3309/4000 [=======================>......]Mon Sep 6 19:29:47 2021[1,0]: - ETA: 1:36:52 - loss: 0.1301 - auc: 0.7771Mon Sep 6 19:29:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:30:47 2021[1,0]:3316/4000 [=======================>......] - ETA: 1:35:54 - loss: 0.1301 - auc: 0.7771Mon Sep 6 19:30:55 2021[1,0]:Mon Sep 6 19:30:38 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:31:47 2021[1,0]:3323/4000 [=======================>......]Mon Sep 6 19:31:47 2021[1,0]: - ETA: 1:34:55 - loss: 0.1300 - auc: 0.7771Mon Sep 6 19:31:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:32:46 2021[1,0]:3330/4000 [=======================>......]Mon Sep 6 19:32:46 2021[1,0]: - ETA: 1:33:56 - loss: 0.1300 - auc: 0.7772Mon Sep 6 19:32:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:33:46 2021[1,0]:3337/4000 [========================>.....]Mon Sep 6 19:33:46 2021[1,0]: - ETA: 1:32:58 - loss: 0.1300 - auc: 0.7772Mon Sep 6 19:33:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3344/4000 [========================>.....]Mon Sep 6 19:34:46 2021[1,0]: - ETA: 1:31:59 - loss: 0.1300 - auc: 0.7772Mon Sep 6 19:34:54 2021[1,0]:Mon Sep 6 19:34:46 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3351/4000 [========================>.....]Mon Sep 6 19:35:46 2021[1,0]: - ETA: 1:31:00 - loss: 0.1300 - auc: 0.7773Mon Sep 6 19:35:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:36:47 2021[1,0]:3358/4000 [========================>.....]Mon Sep 6 19:36:47 2021[1,0]: - ETA: 1:30:02 - loss: 0.1300 - auc: 0.7773Mon Sep 6 19:36:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:37:47 2021[1,0]:3365/4000 [========================>.....]Mon Sep 6 19:37:47 2021[1,0]: - ETA: 1:29:03 - loss: 0.1300 - auc: 0.7773Mon Sep 6 19:37:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:38:46 2021[1,0]:3372/4000 [========================>.....]Mon Sep 6 19:38:46 2021[1,0]: - ETA: 1:28:04 - loss: 0.1300 - auc: 0.7773Mon Sep 6 19:38:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:39:46 2021[1,0]:3379/4000 [========================>.....]Mon Sep 6 19:39:46 2021[1,0]: - ETA: 1:27:05 - loss: 0.1300 - auc: 0.7774Mon Sep 6 19:39:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:40:46 2021[1,0]:3386/4000 [========================>.....]Mon Sep 6 19:40:46 2021[1,0]: - ETA: 1:26:07 - loss: 0.1300 - auc: 0.7774Mon Sep 6 19:40:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:41:46 2021[1,0]:3393/4000 [========================>.....]Mon Sep 6 19:41:46 2021[1,0]: - ETA: 1:25:08 - loss: 0.1299 - auc: 0.7774Mon Sep 6 19:41:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:42:46 2021[1,0]:3400/4000 [========================>.....]Mon Sep 6 19:42:46 2021[1,0]: - ETA: 1:24:09 - loss: 0.1299 - auc: 0.7774Mon Sep 6 19:42:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:43:46 2021[1,0]:3407/4000 [========================>.....]Mon Sep 6 19:43:46 2021[1,0]: - ETA: 1:23:10 - loss: 0.1299 - auc: 0.7774Mon Sep 6 19:43:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:44:46 2021[1,0]:3414/4000 [========================>.....]Mon Sep 6 19:44:46 2021[1,0]: - ETA: 1:22:12 - loss: 0.1299 - auc: 0.7775Mon Sep 6 19:44:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:45:53 2021[1,0]:3422/4000 [========================>.....]Mon Sep 6 19:45:53 2021[1,0]: - ETA: 1:21:04 - loss: 0.1299 - auc: 0.7775Mon Sep 6 19:46:01 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:46:52 2021[1,0]:3429/4000 [========================>.....]Mon Sep 6 19:46:52 2021[1,0]: - ETA: 1:20:05 - loss: 0.1299 - auc: 0.7775Mon Sep 6 19:47:00 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:47:52 2021[1,0]:3436/4000 [========================>.....]Mon Sep 6 19:47:52 2021[1,0]: - ETA: 1:19:07 - loss: 0.1299 - auc: 0.7775Mon Sep 6 19:48:00 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:48:51 2021[1,0]:3443/4000 [========================>.....]Mon Sep 6 19:48:51 2021[1,0]: - ETA: 1:18:08 - loss: 0.1299 - auc: 0.7776Mon Sep 6 19:48:59 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3450/4000 [========================>.....]Mon Sep 6 19:49:49 2021[1,0]: - ETA: 1:17:09 - loss: 0.1299 - auc: 0.7776Mon Sep 6 19:49:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3457/4000 [========================>.....]Mon Sep 6 19:50:47 2021[1,0]: - ETA: 1:16:10 - loss: 0.1298 - auc: 0.7776Mon Sep 6 19:50:55 2021[1,0]:Mon Sep 6 19:50:47 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:51:52 2021[1,0]:3465/4000 [========================>.....]Mon Sep 6 19:51:52 2021[1,0]: - ETA: 1:15:02 - loss: 0.1298 - auc: 0.7777Mon Sep 6 19:52:00 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3472/4000 [=========================>....]Mon Sep 6 19:52:50 2021[1,0]: - ETA: 1:14:03 - loss: 0.1298 - auc: 0.7777Mon Sep 6 19:52:58 2021[1,0]:Mon Sep 6 19:52:41 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3479/4000 [=========================>....]Mon Sep 6 19:53:47 2021[1,0]: - ETA: 1:13:04 - loss: 0.1298 - auc: 0.7777Mon Sep 6 19:53:55 2021[1,0]:Mon Sep 6 19:53:47 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:54:52 2021[1,0]:3487/4000 [=========================>....]Mon Sep 6 19:54:52 2021[1,0]: - ETA: 1:11:56 - loss: 0.1298 - auc: 0.7777Mon Sep 6 19:55:00 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:55:50 2021[1,0]:3494/4000 [=========================>....]Mon Sep 6 19:55:50 2021[1,0]: - ETA: 1:10:57 - loss: 0.1298 - auc: 0.7777Mon Sep 6 19:55:58 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:56:48 2021[1,0]:3501/4000 [=========================>....]Mon Sep 6 19:56:48 2021[1,0]: - ETA: 1:09:58 - loss: 0.1298 - auc: 0.7778Mon Sep 6 19:56:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:57:47 2021[1,0]:3508/4000 [=========================>....]Mon Sep 6 19:57:47 2021[1,0]: - ETA: 1:08:59 - loss: 0.1298 - auc: 0.7778Mon Sep 6 19:57:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:58:46 2021[1,0]:3515/4000 [=========================>....]Mon Sep 6 19:58:46 2021[1,0]: - ETA: 1:08:00 - loss: 0.1298 - auc: 0.7778Mon Sep 6 19:58:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 19:59:52 2021[1,0]:3523/4000 [=========================>....]Mon Sep 6 19:59:52 2021[1,0]: - ETA: 1:06:53 - loss: 0.1298 - auc: 0.7778Mon Sep 6 20:00:01 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:00:49 2021[1,0]:3530/4000 [=========================>....]Mon Sep 6 20:00:49 2021[1,0]: - ETA: 1:05:54 - loss: 0.1297 - auc: 0.7779Mon Sep 6 20:00:58 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:01:48 2021[1,0]:3537/4000 [=========================>....]Mon Sep 6 20:01:48 2021[1,0]: - ETA: 1:04:55 - loss: 0.1297 - auc: 0.7779Mon Sep 6 20:01:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:02:47 2021[1,0]:3544/4000 [=========================>....]Mon Sep 6 20:02:47 2021[1,0]: - ETA: 1:03:56 - loss: 0.1297 - auc: 0.7779Mon Sep 6 20:02:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:03:53 2021[1,0]:3552/4000 [=========================>....]Mon Sep 6 20:03:53 2021[1,0]: - ETA: 1:02:48 - loss: 0.1297 - auc: 0.7780Mon Sep 6 20:04:01 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:04:52 2021[1,0]:3559/4000 [=========================>....]Mon Sep 6 20:04:52 2021[1,0]: - ETA: 1:01:50 - loss: 0.1297 - auc: 0.7780Mon Sep 6 20:05:00 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:05:52 2021[1,0]:3566/4000 [=========================>....]Mon Sep 6 20:05:52 2021[1,0]: - ETA: 1:00:51 - loss: 0.1297 - auc: 0.7780Mon Sep 6 20:06:01 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:06:52 2021[1,0]:3573/4000 [=========================>....]Mon Sep 6 20:06:52 2021[1,0]: - ETA: 59:52 - loss: 0.1297 - auc: 0.7780 Mon Sep 6 20:07:01 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3580/4000 [=========================>....]Mon Sep 6 20:07:52 2021[1,0]: - ETA: 58:53 - loss: 0.1297 - auc: 0.7780Mon Sep 6 20:08:01 2021[1,0]:Mon Sep 6 20:07:52 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3587/4000 [=========================>....]Mon Sep 6 20:08:49 2021[1,0]: - ETA: 57:54 - loss: 0.1297 - auc: 0.7781Mon Sep 6 20:08:57 2021[1,0]:Mon Sep 6 20:08:49 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:09:46 2021[1,0]:3594/4000 [=========================>....]Mon Sep 6 20:09:46 2021[1,0]: - ETA: 56:55 - loss: 0.1297 - auc: 0.7781Mon Sep 6 20:09:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3602/4000 [==========================>...]Mon Sep 6 20:10:52 2021[1,0]: - ETA: 55:48 - loss: 0.1296 - auc: 0.7781Mon Sep 6 20:11:01 2021[1,0]:Mon Sep 6 20:10:36 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3609/4000 [==========================>...]Mon Sep 6 20:11:51 2021[1,0]: - ETA: 54:49 - loss: 0.1296 - auc: 0.7782Mon Sep 6 20:11:59 2021[1,0]:Mon Sep 6 20:11:51 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:12:49 2021[1,0]:3616/4000 [==========================>...]Mon Sep 6 20:12:49 2021[1,0]: - ETA: 53:50 - loss: 0.1296 - auc: 0.7782Mon Sep 6 20:12:58 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:13:49 2021[1,0]:3623/4000 [==========================>...]Mon Sep 6 20:13:49 2021[1,0]: - ETA: 52:51 - loss: 0.1296 - auc: 0.7782Mon Sep 6 20:13:58 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3630/4000 [==========================>...]Mon Sep 6 20:14:48 2021[1,0]: - ETA: 51:52 - loss: 0.1296 - auc: 0.7782Mon Sep 6 20:14:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:15:49 2021[1,0]:3637/4000 [==========================>...]Mon Sep 6 20:15:49 2021[1,0]: - ETA: 50:53 - loss: 0.1296 - auc: 0.7782Mon Sep 6 20:15:58 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:16:49 2021[1,0]:3644/4000 [==========================>...]Mon Sep 6 20:16:49 2021[1,0]: - ETA: 49:55 - loss: 0.1296 - auc: 0.7783Mon Sep 6 20:16:58 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3651/4000 [==========================>...]Mon Sep 6 20:17:49 2021[1,0]: - ETA: 48:56 - loss: 0.1296 - auc: 0.7783Mon Sep 6 20:17:58 2021[1,0]:Mon Sep 6 20:17:49 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3658/4000 [==========================>...]Mon Sep 6 20:18:50 2021[1,0]: - ETA: 47:57 - loss: 0.1296 - auc: 0.7783Mon Sep 6 20:18:59 2021[1,0]:Mon Sep 6 20:18:50 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:19:50 2021[1,0]:3665/4000 [==========================>...]Mon Sep 6 20:19:50 2021[1,0]: - ETA: 46:58 - loss: 0.1296 - auc: 0.7783Mon Sep 6 20:19:58 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:20:49 2021[1,0]:3672/4000 [==========================>...]Mon Sep 6 20:20:49 2021[1,0]: - ETA: 45:59 - loss: 0.1296 - auc: 0.7784Mon Sep 6 20:20:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:21:49 2021[1,0]:3679/4000 [==========================>...]Mon Sep 6 20:21:49 2021[1,0]: - ETA: 45:01 - loss: 0.1295 - auc: 0.7784Mon Sep 6 20:21:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:22:49 2021[1,0]:3686/4000 [==========================>...]Mon Sep 6 20:22:49 2021[1,0]: - ETA: 44:02 - loss: 0.1295 - auc: 0.7784Mon Sep 6 20:22:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3693/4000 [==========================>...]Mon Sep 6 20:23:49 2021[1,0]: - ETA: 43:03 - loss: 0.1295 - auc: 0.7784Mon Sep 6 20:23:58 2021[1,0]:Mon Sep 6 20:23:41 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:24:50 2021[1,0]:3700/4000 [==========================>...]Mon Sep 6 20:24:50 2021[1,0]: - ETA: 42:04 - loss: 0.1295 - auc: 0.7785Mon Sep 6 20:24:59 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3707/4000 [==========================>...]Mon Sep 6 20:25:49 2021[1,0]: - ETA: 41:05 - loss: 0.1295 - auc: 0.7785Mon Sep 6 20:25:58 2021[1,0]:Mon Sep 6 20:25:49 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:26:48 2021[1,0]:3714/4000 [==========================>...]Mon Sep 6 20:26:48 2021[1,0]: - ETA: 40:07 - loss: 0.1295 - auc: 0.7785Mon Sep 6 20:26:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:27:46 2021[1,0]:3721/4000 [==========================>...]Mon Sep 6 20:27:46 2021[1,0]: - ETA: 39:07 - loss: 0.1295 - auc: 0.7785Mon Sep 6 20:27:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:28:54 2021[1,0]:3729/4000 [==========================>...]Mon Sep 6 20:28:54 2021[1,0]: - ETA: 38:00 - loss: 0.1295 - auc: 0.7785Mon Sep 6 20:29:02 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3735/4000 [===========================>..]Mon Sep 6 20:29:46 2021[1,0]: - ETA: 37:10 - loss: 0.1295 - auc: 0.7786Mon Sep 6 20:29:54 2021[1,0]:Mon Sep 6 20:29:46 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3742/4000 [===========================>..] - ETA: 36:11 - loss: 0.1295 - auc: 0.7786Mon Sep 6 20:30:55 2021[1,0]:86Mon Sep 6 20:30:47 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:31:47 2021[1,0]:3749/4000 [===========================>..]Mon Sep 6 20:31:47 2021[1,0]: - ETA: 35:12 - loss: 0.1295 - auc: 0.7786Mon Sep 6 20:31:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:32:47 2021[1,0]:3756/4000 [===========================>..]Mon Sep 6 20:32:47 2021[1,0]: - ETA: 34:13 - loss: 0.1294 - auc: 0.7786Mon Sep 6 20:32:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:33:48 2021[1,0]:3763/4000 [===========================>..]Mon Sep 6 20:33:48 2021[1,0]: - ETA: 33:15 - loss: 0.1294 - auc: 0.7787Mon Sep 6 20:33:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3770/4000 [===========================>..]Mon Sep 6 20:34:49 2021[1,0]: - ETA: 32:16 - loss: 0.1294 - auc: 0.7787Mon Sep 6 20:34:58 2021[1,0]:Mon Sep 6 20:34:49 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:35:50 2021[1,0]:3777/4000 [===========================>..]Mon Sep 6 20:35:50 2021[1,0]: - ETA: 31:17 - loss: 0.1294 - auc: 0.7787Mon Sep 6 20:35:58 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:36:49 2021[1,0]:3784/4000 [===========================>..]Mon Sep 6 20:36:49 2021[1,0]: - ETA: 30:18 - loss: 0.1294 - auc: 0.7787Mon Sep 6 20:36:58 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:37:47 2021[1,0]:3791/4000 [===========================>..]Mon Sep 6 20:37:47 2021[1,0]: - ETA: 29:19 - loss: 0.1294 - auc: 0.7788Mon Sep 6 20:37:55 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3799/4000 [===========================>..]Mon Sep 6 20:38:53 2021[1,0]: - ETA: 28:12 - loss: 0.1294 - auc: 0.7788Mon Sep 6 20:39:02 2021[1,0]:Mon Sep 6 20:38:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3806/4000 [===========================>..]Mon Sep 6 20:39:51 2021[1,0]: - ETA: 27:13 - loss: 0.1294 - auc: 0.7788Mon Sep 6 20:39:59 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:40:50 2021[1,0]:3813/4000 [===========================>..]Mon Sep 6 20:40:50 2021[1,0]: - ETA: 26:14 - loss: 0.1294 - auc: 0.7788Mon Sep 6 20:40:58 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:41:47 2021[1,0]:3820/4000 [===========================>..]Mon Sep 6 20:41:47 2021[1,0]: - ETA: 25:15 - loss: 0.1294 - auc: 0.7788Mon Sep 6 20:41:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:42:54 2021[1,0]:3828/4000 [===========================>..]Mon Sep 6 20:42:54 2021[1,0]: - ETA: 24:07 - loss: 0.1293 - auc: 0.7789Mon Sep 6 20:43:02 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:43:51 2021[1,0]:3835/4000 [===========================>..]Mon Sep 6 20:43:51 2021[1,0]: - ETA: 23:08 - loss: 0.1293 - auc: 0.7789Mon Sep 6 20:44:00 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:44:49 2021[1,0]:3842/4000 [===========================>..]Mon Sep 6 20:44:49 2021[1,0]: - ETA: 22:09 - loss: 0.1293 - auc: 0.7789Mon Sep 6 20:44:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3850/4000 [===========================>..]Mon Sep 6 20:45:54 2021[1,0]: - ETA: 21:02 - loss: 0.1293 - auc: 0.7789Mon Sep 6 20:46:02 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:46:52 2021[1,0]:3857/4000 [===========================>..]Mon Sep 6 20:46:52 2021[1,0]: - ETA: 20:03 - loss: 0.1293 - auc: 0.7790Mon Sep 6 20:47:00 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:47:50 2021[1,0]:3864/4000 [===========================>..]Mon Sep 6 20:47:50 2021[1,0]: - ETA: 19:04 - loss: 0.1293 - auc: 0.7790Mon Sep 6 20:47:59 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:48:49 2021[1,0]:3871/4000 [============================>.]Mon Sep 6 20:48:49 2021[1,0]: - ETA: 18:05 - loss: 0.1293 - auc: 0.7790Mon Sep 6 20:48:58 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:49:48 2021[1,0]:3878/4000 [============================>.]Mon Sep 6 20:49:48 2021[1,0]: - ETA: 17:06 - loss: 0.1293 - auc: 0.7790Mon Sep 6 20:49:56 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:50:54 2021[1,0]:3886/4000 [============================>.]Mon Sep 6 20:50:54 2021[1,0]: - ETA: 15:59 - loss: 0.1293 - auc: 0.7791Mon Sep 6 20:51:02 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3893/4000 [============================>.]Mon Sep 6 20:51:51 2021[1,0]: - ETA: 15:00 - loss: 0.1293 - auc: 0.7791Mon Sep 6 20:52:00 2021[1,0]:Mon Sep 6 20:51:51 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:52:49 2021[1,0]:3900/4000 [============================>.]Mon Sep 6 20:52:49 2021[1,0]: - ETA: 14:01 - loss: 0.1292 - auc: 0.7791Mon Sep 6 20:52:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:53:55 2021[1,0]:3908/4000 [============================>.]Mon Sep 6 20:53:55 2021[1,0]: - ETA: 12:54 - loss: 0.1292 - auc: 0.7791Mon Sep 6 20:54:03 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:54:52 2021[1,0]:3915/4000 [============================>.]Mon Sep 6 20:54:52 2021[1,0]: - ETA: 11:55 - loss: 0.1292 - auc: 0.7791Mon Sep 6 20:55:00 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:55:49 2021[1,0]:3922/4000 [============================>.]Mon Sep 6 20:55:49 2021[1,0]: - ETA: 10:56 - loss: 0.1292 - auc: 0.7792Mon Sep 6 20:55:57 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3930/4000 [============================>.]Mon Sep 6 20:56:53 2021[1,0]: - ETA: 9:48 - loss: 0.1292 - auc: 0.7792Mon Sep 6 20:57:01 2021[1,0]Mon Sep 6 20:57:01 2021[1,0]:>Mon Sep 6 20:56:53 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 20:57:50 2021[1,0]:3937/4000 [============================>.]Mon Sep 6 20:57:50 2021[1,0]: - ETA: 8:49 - loss: 0.1292 - auc: 0.7792Mon Sep 6 20:57:58 2021[1,0]Mon Sep 6 20:57:58 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3944/4000 [============================>.]Mon Sep 6 20:58:47 2021[1,0]: - ETA: 7:51 - loss: 0.1292 - auc: 0.7792Mon Sep 6 20:58:55 2021[1,0]Mon Sep 6 20:58:55 2021[1,0]:Mon Sep 6 20:58:47 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3952/4000 [============================>.]Mon Sep 6 20:59:52 2021[1,0]: - ETA: 6:43 - loss: 0.1292 - auc: 0.7792Mon Sep 6 21:00:00 2021[1,0]Mon Sep 6 21:00:00 2021[1,0]:Mon Sep 6 20:59:44 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 21:00:55 2021[1,0]:3960/4000 [============================>.]Mon Sep 6 21:00:55 2021[1,0]: - ETA: 5:36 - loss: 0.1292 - auc: 0.7793Mon Sep 6 21:01:03 2021[1,0]Mon Sep 6 21:01:03 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 21:01:51 2021[1,0]:3967/4000 [============================>.]Mon Sep 6 21:01:51 2021[1,0]: - ETA: 4:37 - loss: 0.1292 - auc: 0.7793Mon Sep 6 21:01:59 2021[1,0]Mon Sep 6 21:01:59 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 21:02:54 2021[1,0]:3975/4000 [============================>.]Mon Sep 6 21:02:54 2021[1,0]: - ETA: 3:30 - loss: 0.1292 - auc: 0.7793Mon Sep 6 21:03:02 2021[1,0]Mon Sep 6 21:02:54 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3982/4000 [============================>.]Mon Sep 6 21:03:50 2021[1,0]: - ETA: 2:31 - loss: 0.1292 - auc: 0.7793Mon Sep 6 21:03:58 2021[1,0]Mon Sep 6 21:03:58 2021[1,0]:Mon Sep 6 21:03:50 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 21:04:55 2021[1,0]:3990/4000 [============================>.]Mon Sep 6 21:04:55 2021[1,0]: - ETA: 1:24 - loss: 0.1291 - auc: 0.7793Mon Sep 6 21:05:03 2021[1,0]Mon Sep 6 21:05:03 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 21:05:54 2021[1,0]:3997/4000 [============================>.]Mon Sep 6 21:05:54 2021[1,0]: - ETA: 25s - loss: 0.1291 - auc: 0.7793Mon Sep 6 21:06:03 2021[1,0]:::\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 21:06:11 2021[1,0]:3999/4000 [============================>.]Mon Sep 6 21:06:11 2021[1,0]: - ETA: 8s - loss: 0.1291 - auc: 0.7793 Mon Sep 6 21:06:20 2021[1,0]on Sep 6 21:06:11 2021[1,0]:\r" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "[Stage 11:> (0 + 16) / 16]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mon Sep 6 21:09:31 2021[1,0]:4000/4000 [==============================]Mon Sep 6 21:09:31 2021[1,0]: - 33843s 8s/step - loss: 0.1291 - auc: 0.7793 - val_loss: 0.1287 - val_auc: 0.7880 6 21:09:31 2021[1,0]:\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - " \r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Best Loss: 0.128178\n" - ] - } - ], - "source": [ - "main_petastorm()" - ] - }, - { - "cell_type": "markdown", - "id": "49a4020d", - "metadata": {}, - "source": [ - "### Conclusion\n", - "\n", - "From cell[8] and cell[10], we can see the total training time respectively: \n", - "\n", - "- NVTabular: 845s\n", - "- Petastorm: 33843s \n", - "\n", - "So the speedup : 33843 / 845 = `40.05`" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "51361729", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.2" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/dimensions/._SUCCESS.crc b/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/dimensions/._SUCCESS.crc deleted file mode 100644 index 3b7b04493..000000000 Binary files a/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/dimensions/._SUCCESS.crc and /dev/null differ diff --git a/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/dimensions/.part-00000-1bfe973d-db55-449a-8c6a-9aaaa58b0f3e-c000.csv.crc b/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/dimensions/.part-00000-1bfe973d-db55-449a-8c6a-9aaaa58b0f3e-c000.csv.crc deleted file mode 100644 index 33674bbb1..000000000 Binary files a/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/dimensions/.part-00000-1bfe973d-db55-449a-8c6a-9aaaa58b0f3e-c000.csv.crc and /dev/null differ diff --git a/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/dimensions/_SUCCESS b/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/dimensions/_SUCCESS deleted file mode 100644 index e69de29bb..000000000 diff --git a/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/dimensions/part-00000-1bfe973d-db55-449a-8c6a-9aaaa58b0f3e-c000.csv b/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/dimensions/part-00000-1bfe973d-db55-449a-8c6a-9aaaa58b0f3e-c000.csv deleted file mode 100644 index b9e01a929..000000000 --- a/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/dimensions/part-00000-1bfe973d-db55-449a-8c6a-9aaaa58b0f3e-c000.csv +++ /dev/null @@ -1,2 +0,0 @@ -c0,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,c21,c22,c23,c24,c25 -7912888,33822,17138,7338,20045,3,7104,1381,62,5554113,582468,245827,10,2208,10666,103,3,967,14,8165895,2675939,7156452,302515,12021,96,34 diff --git a/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/test_small/._SUCCESS.crc b/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/test_small/._SUCCESS.crc deleted file mode 100644 index 3b7b04493..000000000 Binary files a/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/test_small/._SUCCESS.crc and /dev/null differ diff --git a/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/test_small/.part-00000-39895f81-1e2c-4cb1-a883-b21c9328bf3f-c000.snappy.parquet.crc b/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/test_small/.part-00000-39895f81-1e2c-4cb1-a883-b21c9328bf3f-c000.snappy.parquet.crc deleted file mode 100644 index 72624f728..000000000 Binary files a/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/test_small/.part-00000-39895f81-1e2c-4cb1-a883-b21c9328bf3f-c000.snappy.parquet.crc and /dev/null differ diff --git a/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/test_small/_SUCCESS b/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/test_small/_SUCCESS deleted file mode 100644 index e69de29bb..000000000 diff --git a/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/test_small/part-00000-39895f81-1e2c-4cb1-a883-b21c9328bf3f-c000.snappy.parquet b/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/test_small/part-00000-39895f81-1e2c-4cb1-a883-b21c9328bf3f-c000.snappy.parquet deleted file mode 100644 index 272949e41..000000000 Binary files a/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/test_small/part-00000-39895f81-1e2c-4cb1-a883-b21c9328bf3f-c000.snappy.parquet and /dev/null differ diff --git a/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/train_small/._SUCCESS.crc b/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/train_small/._SUCCESS.crc deleted file mode 100644 index 3b7b04493..000000000 Binary files a/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/train_small/._SUCCESS.crc and /dev/null differ diff --git a/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/train_small/.part-00000-b14c1424-bac6-4e03-9935-e969a5a05f34-c000.snappy.parquet.crc b/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/train_small/.part-00000-b14c1424-bac6-4e03-9935-e969a5a05f34-c000.snappy.parquet.crc deleted file mode 100644 index 3b6f24a55..000000000 Binary files a/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/train_small/.part-00000-b14c1424-bac6-4e03-9935-e969a5a05f34-c000.snappy.parquet.crc and /dev/null differ diff --git a/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/train_small/_SUCCESS b/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/train_small/_SUCCESS deleted file mode 100644 index e69de29bb..000000000 diff --git a/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/train_small/part-00000-b14c1424-bac6-4e03-9935-e969a5a05f34-c000.snappy.parquet b/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/train_small/part-00000-b14c1424-bac6-4e03-9935-e969a5a05f34-c000.snappy.parquet deleted file mode 100644 index 4311ea0d6..000000000 Binary files a/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/train_small/part-00000-b14c1424-bac6-4e03-9935-e969a5a05f34-c000.snappy.parquet and /dev/null differ diff --git a/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/val_small/._SUCCESS.crc b/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/val_small/._SUCCESS.crc deleted file mode 100644 index 3b7b04493..000000000 Binary files a/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/val_small/._SUCCESS.crc and /dev/null differ diff --git a/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/val_small/.part-00000-d22ab04e-dacf-4c66-96cc-2f5cdbaa6177-c000.snappy.parquet.crc b/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/val_small/.part-00000-d22ab04e-dacf-4c66-96cc-2f5cdbaa6177-c000.snappy.parquet.crc deleted file mode 100644 index 1043f863a..000000000 Binary files a/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/val_small/.part-00000-d22ab04e-dacf-4c66-96cc-2f5cdbaa6177-c000.snappy.parquet.crc and /dev/null differ diff --git a/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/val_small/_SUCCESS b/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/val_small/_SUCCESS deleted file mode 100644 index e69de29bb..000000000 diff --git a/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/val_small/part-00000-d22ab04e-dacf-4c66-96cc-2f5cdbaa6177-c000.snappy.parquet b/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/val_small/part-00000-d22ab04e-dacf-4c66-96cc-2f5cdbaa6177-c000.snappy.parquet deleted file mode 100644 index 88793ad65..000000000 Binary files a/examples/ML+DL-Examples/Spark-DL/criteo_train/sample_data/val_small/part-00000-d22ab04e-dacf-4c66-96cc-2f5cdbaa6177-c000.snappy.parquet and /dev/null differ diff --git a/examples/ML+DL-Examples/Spark-DL/criteo_train/spark-env.sh b/examples/ML+DL-Examples/Spark-DL/criteo_train/spark-env.sh deleted file mode 100755 index 58af1b354..000000000 --- a/examples/ML+DL-Examples/Spark-DL/criteo_train/spark-env.sh +++ /dev/null @@ -1,2 +0,0 @@ -SPARK_MASTER_HOST=127.0.0.1 -SPARK_WORKER_OPTS="-Dspark.worker.resource.gpu.amount=16 -Dspark.worker.resource.gpu.discoveryScript=/opt/spark-3.1.2-bin-hadoop3.2/examples/src/main/scripts/getGpusResources.sh" \ No newline at end of file diff --git a/examples/ML+DL-Examples/Spark-DL/criteo_train/start-spark.sh b/examples/ML+DL-Examples/Spark-DL/criteo_train/start-spark.sh deleted file mode 100755 index e4e248e47..000000000 --- a/examples/ML+DL-Examples/Spark-DL/criteo_train/start-spark.sh +++ /dev/null @@ -1,2 +0,0 @@ -/opt/spark-3.1.2-bin-hadoop3.2/sbin/start-master.sh -/opt/spark-3.1.2-bin-hadoop3.2/sbin/start-worker.sh spark://127.0.0.1:7077 diff --git a/examples/ML+DL-Examples/Spark-DL/criteo_train/submit.sh b/examples/ML+DL-Examples/Spark-DL/criteo_train/submit.sh deleted file mode 100755 index aaea59d9b..000000000 --- a/examples/ML+DL-Examples/Spark-DL/criteo_train/submit.sh +++ /dev/null @@ -1,27 +0,0 @@ -SPARK_HOME=/opt/spark-3.1.2-bin-hadoop3.2 -SPARK_URL=spark://127.0.0.1:7077 - -$SPARK_HOME/bin/spark-submit --master $SPARK_URL --deploy-mode client \ ---driver-memory 20G \ ---executor-memory 50G \ ---executor-cores 6 \ ---conf spark.cores.max=96 \ ---conf spark.task.cpus=6 \ ---conf spark.locality.wait=0 \ ---conf spark.yarn.maxAppAttempts=1 \ ---conf spark.sql.shuffle.partitions=4 \ ---conf spark.sql.files.maxPartitionBytes=1024m \ ---conf spark.sql.warehouse.dir=$OUT \ ---conf spark.task.resource.gpu.amount=0.08 \ ---conf spark.executor.resource.gpu.amount=1 \ ---conf spark.executor.resource.gpu.discoveryScript=./getGpusResources.sh \ ---files $SPARK_HOME/examples/src/main/scripts/getGpusResources.sh \ -./criteo_keras.py \ ---num-proc 16 \ ---data-dir file:///data/parquet \ ---logs-dir $PWD/tf_logs \ ---dataloader nvtabular \ ---learning-rate 0.001 \ ---batch-size 65536 \ ---epochs 1 \ ---local-checkpoint-file ckpt_file \ No newline at end of file diff --git a/examples/ML+DL-Examples/Spark-DL/train_inference_e2e/README.md b/examples/ML+DL-Examples/Spark-DL/train_inference_e2e/README.md deleted file mode 100644 index cfeb024aa..000000000 --- a/examples/ML+DL-Examples/Spark-DL/train_inference_e2e/README.md +++ /dev/null @@ -1,29 +0,0 @@ -# Spark DL Training and Inference - -This example is based on the [distributed training example](https://docs.databricks.com/machine-learning/train-model/distributed-training/spark-pytorch-distributor.html#end-to-end-distributed-training-on-databricks-notebook) from Databricks, with additional code to demonstrate: -- how to access datasets on a distributed store, like S3, using Petastorm and/or NVTabular. -- how to conduct parallel inference of a DL model on a Spark cluster, using a new API introduced in Spark 3.4. - -## Run example in Databricks -- Create an AWS S3 bucket to hold the dataset -- Create an AWS IAM User with access keys for the S3 bucket -- Add the AWS secrets to the [Databricks Secrets](https://docs.databricks.com/security/secrets/secrets.html) store using the [databricks CLI](https://docs.databricks.com/dev-tools/cli/index.html) - ``` - databricks secrets create-scope --scope aws-s3 - databricks secrets put --scope aws-s3 --key aws-access-key - databricks secrets put --scope aws-s3 --key aws-secret-key - ``` - **NOTE**: Do not hard-code the values of the secrets anywhere and avoid accidental logging of the values. - -- Copy the [init.sh](./init.sh) script to DBFS using the - ``` - databricks fs cp init.sh dbfs:/path/to/init.sh - ``` -- Create a multi-node Databricks cluster with the following configuration: - - Databricks Runtime Versions: `13.1 ML GPU Runtime` - - Worker type: `g4dn.xlarge` (x2) - - Driver type: `same as worker` - - Advanced options -> Init Scripts: `dbfs:/path/to/init.sh` -- Start the cluster -- Import the notebook into your Databricks workspace -- Attach the cluster to your notebook and run the notebook diff --git a/examples/ML+DL-Examples/Spark-DL/train_inference_e2e/init.sh b/examples/ML+DL-Examples/Spark-DL/train_inference_e2e/init.sh deleted file mode 100755 index 7800b2130..000000000 --- a/examples/ML+DL-Examples/Spark-DL/train_inference_e2e/init.sh +++ /dev/null @@ -1,15 +0,0 @@ -!/bin/bash - -# install dependencies for petastorm and nvtabular data loaders -/databricks/python/bin/pip install cudf-cu11 dask-cudf-cu11 --extra-index-url=https://pypi.nvidia.com -/databricks/python/bin/pip install \ - merlin-dataloader \ - nvtabular \ - "numpy<1.24" \ - "protobuf<3.20" \ - "pyarrow==10.0.1" \ - s3fs \ - tritonclient - -# https://stackoverflow.com/questions/71759248/importerror-cannot-import-name-builder-from-google-protobuf-internal -wget https://raw.githubusercontent.com/protocolbuffers/protobuf/main/python/google/protobuf/internal/builder.py -O /databricks/python/lib/python3.10/site-packages/google/protobuf/internal/builder.py diff --git a/examples/ML+DL-Examples/Spark-DL/train_inference_e2e/spark_e2e.ipynb b/examples/ML+DL-Examples/Spark-DL/train_inference_e2e/spark_e2e.ipynb deleted file mode 100644 index 61bcda00f..000000000 --- a/examples/ML+DL-Examples/Spark-DL/train_inference_e2e/spark_e2e.ipynb +++ /dev/null @@ -1,1268 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "18a99ee7-1b0d-447b-a27c-b46b0903af68", - "showTitle": false, - "title": "" - } - }, - "source": [ - "# Distributed Training and Inference with Spark 3.4\n", - "\n", - "Based on: https://docs.databricks.com/machine-learning/train-model/distributed-training/spark-pytorch-distributor.html#end-to-end-distributed-training-on-databricks-notebook" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "473535d4-472c-483f-897a-2a696f237989", - "showTitle": false, - "title": "" - } - }, - "source": [ - "## Setup" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "8abda0c7-147e-4485-afc6-6d16ac84ddb1", - "showTitle": false, - "title": "" - } - }, - "outputs": [], - "source": [ - "access_key = dbutils.secrets.get(scope=\"aws-s3\", key = \"aws-access-key\")\n", - "secret_key = dbutils.secrets.get(scope=\"aws-s3\", key = \"aws-secret-key\")\n", - "sc._jsc.hadoopConfiguration().set(\"fs.s3a.access.key\", access_key)\n", - "sc._jsc.hadoopConfiguration().set(\"fs.s3a.secret.key\", secret_key)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "b0c52671-2deb-4e9f-980d-a321fb586e8d", - "showTitle": false, - "title": "" - } - }, - "outputs": [], - "source": [ - "S3_BUCKET=\"s3a://path/to/bucket\"" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "a505372a-d7f5-469c-8f17-15376fecc077", - "showTitle": false, - "title": "" - } - }, - "source": [ - "### Define Model" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "81ef8262-a624-4f9e-81ea-9f193e224f91", - "showTitle": false, - "title": "" - } - }, - "outputs": [], - "source": [ - "import os\n", - "import torch\n", - "import torch.nn as nn\n", - "import torch.nn.functional as F\n", - "import torch.optim as optim\n", - "from time import time\n", - "from torchvision import datasets, transforms\n", - "\n", - "PYTORCH_DIR = \"/dbfs/ml/pytorch\"\n", - "os.environ[\"TORCH_HOME\"] = PYTORCH_DIR\n", - "\n", - "batch_size = 100\n", - "num_epochs = 5\n", - "momentum = 0.5\n", - "log_interval = 100\n", - "learning_rate = 0.001\n", - "\n", - "\n", - "# Model definition\n", - "class Net(nn.Module):\n", - " def __init__(self):\n", - " super(Net, self).__init__()\n", - " self.conv1 = nn.Conv2d(1, 10, kernel_size=5)\n", - " self.conv2 = nn.Conv2d(10, 20, kernel_size=5)\n", - " self.conv2_drop = nn.Dropout2d()\n", - " self.fc1 = nn.Linear(320, 50)\n", - " self.fc2 = nn.Linear(50, 10)\n", - "\n", - " def forward(self, x):\n", - " x = F.relu(F.max_pool2d(self.conv1(x), 2))\n", - " x = F.relu(F.max_pool2d(self.conv2_drop(self.conv2(x)), 2))\n", - " x = x.view(-1, 320)\n", - " x = F.relu(self.fc1(x))\n", - " x = F.dropout(x, training=self.training)\n", - " x = self.fc2(x)\n", - " return F.log_softmax(x, dim=-1)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "8accd126-2ac5-41d6-a53f-18d53ef91e3d", - "showTitle": false, - "title": "" - } - }, - "source": [ - "### Define Common Functions" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "9aa51c1b-d55e-4718-8e35-0773de29e1f9", - "showTitle": false, - "title": "" - } - }, - "outputs": [], - "source": [ - "def prepare_batch(batch):\n", - " # shim to allow transforming the output of the data loader in Advanced section below\n", - " # pass-through by default\n", - " return batch\n", - "\n", - "def train_one_epoch(model, device, data_loader, optimizer, epoch, prepare_batch):\n", - " model.train()\n", - " for batch_idx, batch in enumerate(data_loader):\n", - " data, target = prepare_batch(batch) # added prepare_batch for advanced data loading\n", - " data, target = data.to(device), target.to(device)\n", - " optimizer.zero_grad()\n", - " output = model(data)\n", - " loss = F.nll_loss(output, target)\n", - " loss.backward()\n", - " optimizer.step()\n", - " if batch_idx % log_interval == 0:\n", - " print(\n", - " \"Train Epoch: {} [{}]\\tLoss: {:.6f}\".format(\n", - " epoch,\n", - " batch_idx * len(data),\n", - " loss.item(),\n", - " )\n", - " )\n", - "\n", - "\n", - "def save_checkpoint(log_dir, model, optimizer, epoch):\n", - " filepath = log_dir + \"/checkpoint-{epoch}.pth.tar\".format(epoch=epoch)\n", - " state = {\n", - " \"model\": model.module.state_dict(),\n", - " \"optimizer\": optimizer.state_dict(),\n", - " }\n", - " torch.save(state, filepath)\n", - "\n", - "\n", - "def load_checkpoint(log_dir, epoch=num_epochs):\n", - " filepath = log_dir + \"/checkpoint-{epoch}.pth.tar\".format(epoch=epoch)\n", - " return torch.load(filepath)\n", - "\n", - "\n", - "def create_log_dir():\n", - " log_dir = os.path.join(PYTORCH_DIR, str(time()))\n", - " os.makedirs(log_dir)\n", - " return log_dir" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "3fc950d9-72c4-4b46-a23f-a4912132aa1a", - "showTitle": false, - "title": "" - } - }, - "source": [ - "## Distributed Training" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "772ce27d-ed0c-4909-b41f-31f60d0a9d0d", - "showTitle": false, - "title": "" - } - }, - "source": [ - "### Main Function\n", - "\n", - "The [conversion steps](https://pytorch.org/tutorials/intermediate/ddp_series_multinode.html) from single-node to distributed are highlighted in comments below.\n", - "\n", - "**Note**: the data loaders have been factored out, so we can swap in different implementations in the \"Advanced\" section below." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "64fef046-c79e-4db0-b413-68d41a487374", - "showTitle": false, - "title": "" - } - }, - "outputs": [], - "source": [ - "def main_fn(directory):\n", - " #### Added Torch Distributed imports ####\n", - " import torch.distributed as dist\n", - " from torch.nn.parallel import DistributedDataParallel as DDP\n", - "\n", - " print(\"Running distributed training\")\n", - " dist.init_process_group(\"nccl\")\n", - "\n", - " local_rank = int(os.environ[\"LOCAL_RANK\"])\n", - " global_rank = int(os.environ[\"RANK\"])\n", - " world_size = int(os.environ[\"WORLD_SIZE\"])\n", - "\n", - " #### Adjust batch size per worker ####\n", - " adjusted_batch_size = batch_size // world_size\n", - " print(f\"{adjusted_batch_size=}\")\n", - "\n", - " if global_rank == 0:\n", - " train_parameters = {\n", - " \"batch_size\": adjusted_batch_size,\n", - " \"epochs\": num_epochs,\n", - " \"trainer\": \"TorchDistributor\",\n", - " }\n", - "\n", - " model = Net().to(local_rank)\n", - " #### Added Distributed Model ####\n", - " ddp_model = DDP(model, device_ids=[local_rank], output_device=local_rank)\n", - "\n", - " optimizer = optim.SGD(ddp_model.parameters(), lr=learning_rate, momentum=momentum)\n", - " for epoch in range(1, num_epochs + 1):\n", - " # train data_loader\n", - " data_loader = get_train_dataloader(adjusted_batch_size)\n", - "\n", - " train_one_epoch(ddp_model, local_rank, data_loader, optimizer, epoch, prepare_batch)\n", - "\n", - " if global_rank == 0:\n", - " save_checkpoint(directory, ddp_model, optimizer, epoch)\n", - "\n", - " if global_rank == 0:\n", - " ddp_model.eval()\n", - "\n", - " # test data_loader\n", - " data_loader = get_test_dataloader(batch_size)\n", - "\n", - " test_loss = 0\n", - " num_batches = 0\n", - " for batch in data_loader:\n", - " data, target = prepare_batch(batch)\n", - " device = torch.device(\"cuda\")\n", - " data, target = data.to(device), target.to(device)\n", - " output = ddp_model(data)\n", - " test_loss += F.nll_loss(output, target)\n", - " num_batches += 1\n", - "\n", - " test_loss /= num_batches\n", - " print(\"Average test loss: {}\".format(test_loss.item()))\n", - "\n", - " dist.destroy_process_group()\n", - "\n", - " return \"finished\" # can return any picklable object" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "129e940f-3b35-457e-9727-3caa3de34baf", - "showTitle": false, - "title": "" - } - }, - "source": [ - "### Torch Datasets Data Loader\n", - "This example just downloads the MNIST dataset to each worker and then uses the Torch DistributedSampler to shard the data.\n", - "\n", - "**Note**: that this only works if the dataset is small enough to be downloaded to each worker. For larger datasets, see the \"Advanced\" section below." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "5def7d8d-69f9-47ca-8a21-ba31a9a4b6a9", - "showTitle": false, - "title": "" - } - }, - "outputs": [], - "source": [ - "def get_train_dataloader(batch_size, distributed=True):\n", - " if distributed:\n", - " from torch.utils.data.distributed import DistributedSampler\n", - "\n", - " train_dataset = datasets.MNIST(\n", - " \"data\",\n", - " train=True,\n", - " download=True,\n", - " transform=transforms.Compose(\n", - " [transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,))]\n", - " ),\n", - " )\n", - " \n", - " #### Added Distributed Dataloader ####\n", - " train_sampler = DistributedSampler(train_dataset) if distributed else None\n", - " data_loader = torch.utils.data.DataLoader(\n", - " train_dataset, batch_size=batch_size, sampler=train_sampler\n", - " )\n", - " return data_loader\n", - "\n", - "def get_test_dataloader(batch_size):\n", - " test_dataset = datasets.MNIST(\n", - " \"data\",\n", - " train=False,\n", - " download=True,\n", - " transform=transforms.Compose(\n", - " [transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,))]\n", - " ),\n", - " )\n", - " data_loader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size) \n", - " return data_loader" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "50af6897-8441-471e-bc76-513008b964f2", - "showTitle": false, - "title": "" - } - }, - "source": [ - "### Train with TorchDistributor\n", - "See: https://spark.apache.org/docs/latest//api/python/reference/api/pyspark.ml.torch.distributor.TorchDistributor.html" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "ffee450c-b4b7-4606-8d02-29807a5535e4", - "showTitle": false, - "title": "" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Log directory: /dbfs/ml/pytorch/1683755013.639396\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Started distributed training with 2 executor proceses\n", - "Running distributed training\n", - "Running distributed training\n", - "adjusted_batch_size=50\n", - "adjusted_batch_size=50\n", - "Train Epoch: 1 [0]\tLoss: 2.315846\n", - "Train Epoch: 1 [0]\tLoss: 2.328258\n", - "Train Epoch: 1 [5000]\tLoss: 2.318942\n", - "Train Epoch: 1 [5000]\tLoss: 2.305338\n", - "Train Epoch: 1 [10000]\tLoss: 2.295995\n", - "Train Epoch: 1 [10000]\tLoss: 2.291051\n", - "Train Epoch: 1 [15000]\tLoss: 2.282527\n", - "Train Epoch: 1 [15000]\tLoss: 2.305161\n", - "Train Epoch: 1 [20000]\tLoss: 2.282576\n", - "Train Epoch: 1 [20000]\tLoss: 2.280523\n", - "Train Epoch: 1 [25000]\tLoss: 2.272546\n", - "Train Epoch: 1 [25000]\tLoss: 2.324517\n", - "Train Epoch: 2 [0]\tLoss: 2.263000\n", - "Train Epoch: 2 [0]\tLoss: 2.253219\n", - "Train Epoch: 2 [5000]\tLoss: 2.269941\n", - "Train Epoch: 2 [5000]\tLoss: 2.256959\n", - "Train Epoch: 2 [10000]\tLoss: 2.222101\n", - "Train Epoch: 2 [10000]\tLoss: 2.250984\n", - "Train Epoch: 2 [15000]\tLoss: 2.173892\n", - "Train Epoch: 2 [15000]\tLoss: 2.258744\n", - "Train Epoch: 2 [20000]\tLoss: 2.161367\n", - "Train Epoch: 2 [20000]\tLoss: 2.194634\n", - "Train Epoch: 2 [25000]\tLoss: 2.214464\n", - "Train Epoch: 2 [25000]\tLoss: 2.104484\n", - "Train Epoch: 3 [0]\tLoss: 2.073377\n", - "Train Epoch: 3 [0]\tLoss: 2.065120\n", - "Train Epoch: 3 [5000]\tLoss: 2.067660\n", - "Train Epoch: 3 [5000]\tLoss: 2.036174\n", - "Train Epoch: 3 [10000]\tLoss: 1.958426\n", - "Train Epoch: 3 [10000]\tLoss: 1.914909\n", - "Train Epoch: 3 [15000]\tLoss: 1.715065\n", - "Train Epoch: 3 [15000]\tLoss: 1.934866\n", - "Train Epoch: 3 [20000]\tLoss: 1.745061\n", - "Train Epoch: 3 [20000]\tLoss: 1.685388\n", - "Train Epoch: 3 [25000]\tLoss: 1.668422\n", - "Train Epoch: 3 [25000]\tLoss: 1.605796\n", - "Train Epoch: 4 [0]\tLoss: 1.664934\n", - "Train Epoch: 4 [0]\tLoss: 1.377108\n", - "Train Epoch: 4 [5000]\tLoss: 1.495357\n", - "Train Epoch: 4 [5000]\tLoss: 1.586019\n", - "Train Epoch: 4 [10000]\tLoss: 1.228206\n", - "Train Epoch: 4 [10000]\tLoss: 1.422787\n", - "Train Epoch: 4 [15000]\tLoss: 1.213615\n", - "Train Epoch: 4 [15000]\tLoss: 1.411315\n", - "Train Epoch: 4 [20000]\tLoss: 1.170620\n", - "Train Epoch: 4 [20000]\tLoss: 1.348655\n", - "Train Epoch: 4 [25000]\tLoss: 1.186718\n", - "Train Epoch: 4 [25000]\tLoss: 1.093435\n", - "Train Epoch: 5 [0]\tLoss: 1.262350\n", - "Train Epoch: 5 [0]\tLoss: 1.168563\n", - "Train Epoch: 5 [5000]\tLoss: 1.026802\n", - "Train Epoch: 5 [5000]\tLoss: 1.100900\n", - "Train Epoch: 5 [10000]\tLoss: 1.011708\n", - "Train Epoch: 5 [10000]\tLoss: 0.810456\n", - "Train Epoch: 5 [15000]\tLoss: 0.761695\n", - "Train Epoch: 5 [15000]\tLoss: 0.966266\n", - "Train Epoch: 5 [20000]\tLoss: 1.110494\n", - "Train Epoch: 5 [20000]\tLoss: 0.952292\n", - "Train Epoch: 5 [25000]\tLoss: 0.948426\n", - "Train Epoch: 5 [25000]\tLoss: 0.963606\n", - "Average test loss: 0.4790334701538086\n", - "Finished distributed training with 2 executor proceses\n" - ] - } - ], - "source": [ - "from pyspark.ml.torch.distributor import TorchDistributor\n", - "\n", - "multi_node_multi_gpu_dir = create_log_dir()\n", - "print(\"Log directory: \", create_log_dir())\n", - "\n", - "# Set num_processes = NUM_WORKERS * NUM_GPUS_PER_WORKER\n", - "output_dist = TorchDistributor(num_processes=2, local_mode=False, use_gpu=True).run(main_fn, multi_node_multi_gpu_dir)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "a40e7c6d-f486-4142-9954-62a88cfaece8", - "showTitle": false, - "title": "" - } - }, - "source": [ - "## Distributed Inference" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "2dbd74bc-523f-4d80-9187-2546eb20063f", - "showTitle": false, - "title": "" - } - }, - "source": [ - "### Convert Test Data to Parquet\n", - "\n", - "To emulate the use case where the dataset resides on a distributed file system, save the dataset as Parquet files to an S3 bucket.\n", - "\n", - "**Note**: this only needs to be run once, so you can skip this section if you've already generated the test dataset on S3." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "79573819-64b1-41cb-a6f1-f3dca07f7351", - "showTitle": false, - "title": "" - } - }, - "outputs": [ - { - "data": { - "application/vnd.databricks.v1+bamboolib_hint": "{\"pd.DataFrames\": [], \"version\": \"0.0.1\"}", - "text/plain": [] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "import numpy as np\n", - "import pandas as pd\n", - "from pyspark.sql.types import StructType, StructField, ArrayType, FloatType, LongType" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "754362a2-ad00-45f7-86bc-4a24e85281c0", - "showTitle": false, - "title": "" - } - }, - "outputs": [], - "source": [ - "def save_as_parquet(data_loader, bucket):\n", - " data, labels = next(iter(data_loader))\n", - "\n", - " # flatten images\n", - " np_data = np.array(data).reshape(-1, 784)\n", - "\n", - " # create pandas dataframe with single array column\n", - " pdf = pd.DataFrame(np_data)\n", - " pdf1 = pd.DataFrame()\n", - " pdf1['data'] = pdf.values.tolist()\n", - " pdf1['label'] = labels\n", - " \n", - " # convert to Spark DataFrame\n", - " schema = StructType([\n", - " StructField(\"data\",ArrayType(FloatType()), True),\n", - " StructField(\"label\", LongType(), True)\n", - " ])\n", - " df = spark.createDataFrame(pdf1, schema)\n", - "\n", - " # save DataFrame to S3 bucket as parquet\n", - " df.write.mode(\"overwrite\").parquet(bucket)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "de27976f-96de-44c6-931c-7467bc63d16f", - "showTitle": false, - "title": "" - } - }, - "outputs": [], - "source": [ - "# save test dataset as parquet to S3\n", - "test_data_loader = get_test_dataloader(batch_size=10000)\n", - "save_as_parquet(test_data_loader, f\"{S3_BUCKET}/test\")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "3815be25-0e4f-4a6a-8348-50922c75375a", - "showTitle": false, - "title": "" - } - }, - "source": [ - "### Inference with predict_batch_udf\n", - "See: https://spark.apache.org/docs/latest//api/python/reference/api/pyspark.ml.functions.predict_batch_udf.html" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "28293bff-2e66-4a16-80d8-1f591a1db981", - "showTitle": false, - "title": "" - } - }, - "outputs": [], - "source": [ - "from pyspark.ml.functions import predict_batch_udf\n", - "from pyspark.sql.functions import struct, col, array\n", - "from pyspark.sql.types import ArrayType, FloatType, Union, Dict" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "ce0837d6-2b18-4553-8803-f8cb773b3ef7", - "showTitle": false, - "title": "" - } - }, - "outputs": [], - "source": [ - "df = spark.read.parquet(\"${S3_BUCKET}/test\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "f8d8af05-9cc4-4c30-989a-ab917e56b3e2", - "showTitle": false, - "title": "" - } - }, - "outputs": [], - "source": [ - "def predict_batch_fn():\n", - " import torch \n", - " device = torch.device(\"cuda\")\n", - " model = Net().to(device)\n", - " checkpoint = load_checkpoint(multi_node_multi_gpu_dir)\n", - " model.load_state_dict(checkpoint['model'])\n", - "\n", - " def predict(inputs):\n", - " torch_inputs = torch.from_numpy(inputs).to(device)\n", - " outputs = model(torch_inputs)\n", - " return outputs.cpu().detach().numpy()\n", - " \n", - " return predict" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "9f3b8672-7e84-4c7c-8020-3d552e30b934", - "showTitle": false, - "title": "" - } - }, - "outputs": [], - "source": [ - "mnist = predict_batch_udf(predict_batch_fn,\n", - " input_tensor_shapes=[[1,28,28]],\n", - " return_type=ArrayType(FloatType()),\n", - " batch_size=1000)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "7ce2e38c-2dfd-4daa-950b-f928a39cb874", - "showTitle": false, - "title": "" - } - }, - "outputs": [], - "source": [ - "preds = df.withColumn(\"preds\", mnist('data')).collect()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "a77c5663-c109-403d-a7d9-d1d967f53e25", - "showTitle": false, - "title": "" - } - }, - "source": [ - "### Check predictions" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "56e06323-5694-4cc6-be8c-ae643c19b30d", - "showTitle": false, - "title": "" - } - }, - "outputs": [], - "source": [ - "import matplotlib.pyplot as plt\n", - "import numpy as np" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "1dbdda94-9726-49da-8734-9bfde8bbb9d0", - "showTitle": false, - "title": "" - } - }, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAA9EAAAIICAYAAABpdm0gAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAA9hAAAPYQGoP6dpAABgo0lEQVR4nO3de3xU1bn/8W/uhCSEW0gIN8NVBQVBQEQRJEKhWFG0am0L2FaUi1xOveCpcqoesWoVxXhtC9qjxXL8IRURqwgoykUjtHIVFRSFBGhJgkhCklm/PzgE414DezKTzGV/3q/X/JFn1qy9dngWM8/syTNxxhgjAAAAAABwSvHhXgAAAAAAANGCIhoAAAAAAJcoogEAAAAAcIkiGgAAAAAAlyiiAQAAAABwiSIaAAAAAACXKKIBAAAAAHCJIhoAAAAAAJcoogEAAAAAcIkiOoKcdtppGjduXM3PK1euVFxcnFauXBmyY8TFxem//uu/QjYfEErsAXgZ+Q+vYw/Ay8j/6EIR/X/mz5+vuLi4mlujRo3UtWtXTZ48WcXFxeFeXkCWLl0aVRvk8ccf1xlnnKGUlBS1adNGM2bM0OHDh8O9LM9hD4QPeyD8yP/w2bp1q37wgx8oPT1dzZs3189+9jPt378/3MvyHPZAeDz77LO66KKLlJ2drZSUFOXl5Wn8+PHatWtXuJfmKeR/+FVWVurMM89UXFycHnrooXAv55QSw72ASHP33XcrLy9P5eXlWr16tZ588kktXbpUmzZtUuPGjRt0LYMGDdKRI0eUnJwc0OOWLl2qgoIC6wY6cuSIEhMj55/9tttu0wMPPKArr7xSU6dO1ZYtWzR37lxt3rxZb7zxRriX50nsgYbFHogs5H/D+uqrrzRo0CBlZmbqvvvu0zfffKOHHnpIH3/8sdavXx/wuSN47IGGtWHDBuXl5elHP/qRmjVrpp07d+rZZ5/VkiVL9I9//EO5ubnhXqKnkP/hM3fuXH355ZfhXoZrkflbDKMRI0bo3HPPlST98pe/VIsWLfTwww9r8eLFuvbaa62POXz4sNLS0kK+lvj4eDVq1Cikc4Z6vmDs3btXDz/8sH72s5/p+eefr4l37dpVU6ZM0auvvqpLL700jCv0JvZAw2EPRB7yv2Hdd999Onz4sAoLC9W+fXtJUr9+/XTJJZdo/vz5uuGGG8K8Qu9hDzSsJ554whEbPXq0zj33XD3//PO6/fbbw7Aq7yL/w2Pfvn26++67ddttt+muu+4K93Jc4ePcp3DxxRdLknbu3ClJGjdunNLT0/XZZ59p5MiRysjI0HXXXSdJ8vl8mjNnjrp3765GjRopOztbEyZM0MGDB2vNaYzRvffeq7Zt26px48YaMmSINm/e7Di2v7+FWLdunUaOHKlmzZopLS1NZ599th599NGa9RUUFEhSrY+lHGf7W4gNGzZoxIgRatKkidLT0zV06FCtXbu21pjjH3N57733NGPGDGVlZSktLU2XX36542N3paWl2rZtm0pLS0/6u12zZo2qqqp0zTXX1Iof/3nBggUnfTwaBnvgGPaAN5H/x9RH/kvSyy+/rFGjRtUU0JKUn5+vrl276q9//espH4/6xx44pr72gM1pp50mSSopKanT4xE65P8x9Z3/t99+u7p166af/vSnrh8TblyJPoXPPvtMktSiRYuaWFVVlYYPH64LLrhADz30UM3HOyZMmKD58+dr/Pjxuvnmm7Vz5049/vjj2rBhg9577z0lJSVJku666y7de++9GjlypEaOHKmPPvpIw4YN09GjR0+5njfffFOjRo1S69atNXXqVOXk5Gjr1q1asmSJpk6dqgkTJmjPnj1688039ec///mU823evFkXXnihmjRpoltvvVVJSUl6+umnNXjwYK1atUr9+/evNX7KlClq1qyZZs2apV27dmnOnDmaPHmyXnrppZoxixYt0vjx4zVv3rxaDRK+r6KiQpKUmppaK37891lYWHjK9aP+sQfYA15G/tdf/n/99dfat29fzVWf7+rXr5+WLl16yvWj/rEH6m8PfNe//vUvVVdX68svv9Tdd98tSRo6dKirx6L+kP/1n//r16/Xc889p9WrV9cq+COegTHGmHnz5hlJ5q233jL79+83u3fvNgsWLDAtWrQwqamp5quvvjLGGDN27Fgjydx+++21Hv/uu+8aSeaFF16oFV+2bFmt+L59+0xycrL54Q9/aHw+X824O+64w0gyY8eOrYmtWLHCSDIrVqwwxhhTVVVl8vLyTIcOHczBgwdrHee7c02aNMn4+6eVZGbNmlXz8+jRo01ycrL57LPPamJ79uwxGRkZZtCgQY7fT35+fq1jTZ8+3SQkJJiSkhLH2Hnz5lnXcFxhYaGRZO65555a8eO/s/T09JM+HqHFHmAPeBn53/D5/8EHHxhJ5vnnn3fcd8sttxhJpry8/KRzIHTYAw2/B74rJSXFSDKSTIsWLcxjjz3m+rEIHvkfnvz3+XymX79+5tprrzXGGLNz504jyTz44IOnfGy48XHu78nPz1dWVpbatWuna665Runp6Vq0aJHatGlTa9xNN91U6+eFCxcqMzNTl1xyiQ4cOFBz69Onj9LT07VixQpJ0ltvvaWjR49qypQptd5tmTZt2inXtmHDBu3cuVPTpk1T06ZNa91Xl3duqqur9fe//12jR49Wx44da+KtW7fWT37yE61evVplZWW1HnPDDTfUOtaFF16o6upqffHFFzWxcePGyRhzynefevfurf79++t3v/ud5s2bp127dun111/XhAkTlJSUpCNHjgR8Tggee4A94GXkf8Pl//H8TklJcdx3/O/22AMNjz3QcHvgu15//XUtXbpUv//979W+fXu+oSFMyP+Gzf/58+fr448/1u9+97uA1x9ufJz7ewoKCtS1a1clJiYqOztb3bp1U3x87fcaEhMT1bZt21qxHTt2qLS0VK1atbLOu2/fPkmqSbIuXbrUuj8rK0vNmjU76dqOf6SkR48e7k/oJPbv369vv/1W3bp1c9x3xhlnyOfzaffu3erevXtN/Lt/tyapZs3f/3sPt15++WVdffXVuv766yVJCQkJmjFjhlatWqXt27fXaU4Ehz1wDHvAm8j/Yxoi/4//GcPxP2v4rvLy8lpj0HDYA8c01HPAcUOGDJF0rLHVZZddph49eig9PV2TJ08Oal4Ehvw/piHyv6ysTDNnztQtt9yidu3aBfz4cKOI/p5+/fpZ/z7ru1JSUhwbyufzqVWrVnrhhResj8nKygrZGsMpISHBGjfG1Gm+Nm3aaPXq1dqxY4eKiorUpUsX5eTkKDc3V127dg1mqagj9sDJsQdiG/l/cqHM/9atW0s61qX++/bu3avmzZtbr1KjfrEHTi7UzwE2nTp10jnnnKMXXniBIrqBkf8nF8r8f+ihh3T06FFdffXVNd+L/tVXX0k6VpTv2rVLubm5EftVhxTRIdKpUye99dZbGjhw4EnfOe/QoYOkY+9YffejE/v37z/luzidOnWSJG3atEn5+fl+x7n9SEdWVpYaN25svdq1bds2xcfHN9g7Q126dKl5V27Lli3au3dvQB+DQvixB4LDHohu5H/g2rRpo6ysLH344YeO+9avX69evXrV27EReuyB0Dpy5Ij1UxqITOR/4L788ksdPHiw1pXu4+677z7dd9992rBhQ8Q+F/A30SHy4x//WNXV1brnnnsc91VVVdV8TUF+fr6SkpI0d+7cWu/azJkz55TH6N27t/Ly8jRnzhzH1x58d67j31V3qq9GSEhI0LBhw7R48eKad4Akqbi4WC+++KIuuOACNWnS5JTr+r5gvtrB5/Pp1ltvVePGjXXjjTcG/HiED3vgBPaA95D/JwSS/2PGjNGSJUu0e/fumtjy5cv1ySef6Kqrrgr42Agf9sAJbvdAVVWVtXBav369Pv7441NeEUXkIP9PcJv/N998sxYtWlTr9vTTT0s69nfVixYtUl5eXsDHbyhciQ6Riy66SBMmTNDs2bO1ceNGDRs2TElJSdqxY4cWLlyoRx99VFdeeaWysrL061//WrNnz9aoUaM0cuRIbdiwQa+//rpatmx50mPEx8frySef1KWXXqpevXpp/Pjxat26tbZt26bNmzfrjTfekCT16dNH0rHkHD58uBISEhzfQ3vcvffeqzfffFMXXHCBJk6cqMTERD399NOqqKjQAw88UKffRSCt7adOnary8nL16tVLlZWVevHFF2ta3X//7y4Q2dgDJ7AHvIf8PyGQ/L/jjju0cOFCDRkyRFOnTtU333yjBx98UGeddZbGjx9fp+MjPNgDJ7jdA998843atWunq6++Wt27d1daWpo+/vhjzZs3T5mZmbrzzjvrdHw0PPL/BLf537t3b/Xu3btW7Hgx3717d40ePbpOx28wDdsMPHIdb8f+wQcfnHTc2LFjTVpamt/7n3nmGdOnTx+TmppqMjIyzFlnnWVuvfVWs2fPnpox1dXV5re//a1p3bq1SU1NNYMHDzabNm0yHTp0OGlr++NWr15tLrnkEpORkWHS0tLM2WefbebOnVtzf1VVlZkyZYrJysoycXFxtdrc63ut7Y0x5qOPPjLDhw836enppnHjxmbIkCHm/fffd/X7sa0xkNb28+bNMz179jRpaWkmIyPDDB061Lz99tunfBxCjz3AHvAy8j88+W+MMZs2bTLDhg0zjRs3Nk2bNjXXXXedKSoqcvVYhA57oOH3QEVFhZk6dao5++yzTZMmTUxSUpLp0KGD+cUvfmF27tx50scitMj/8D0HfFc0fcVVnDEh7IQAAAAAAEAM42+iAQAAAABwiSIaAAAAAACXKKIBAAAAAHCJIhoAAAAAAJcoogEAAAAAcIkiGgAAAAAAlxLra+KCggI9+OCDKioqUs+ePTV37lz169fvlI/z+Xzas2ePMjIyFBcXV1/LA/wyxujQoUPKzc1VfHzd3meqa/5L7AGEVyjyX+I5ANGL5wB4GfkPLwso/+vjy6cXLFhgkpOTzZ/+9CezefNm86tf/co0bdrUFBcXn/Kxu3fvNpK4cQv7bffu3Q2e/+wBbpFyq2v+B7sHyH9ukXLjOYCbl2/kPzcv39zkf5wxxijE+vfvr759++rxxx+XdOxdpXbt2mnKlCm6/fbbT/rY0tJSNW3aVBdopBKVFOqlAadUpUqt1lKVlJQoMzMz4McHk/8SewDhFWz+SzwHILrxHAAvI//hZYHkf8g/zn306FEVFhZq5syZNbH4+Hjl5+drzZo1jvEVFRWqqKio+fnQoUP/t7AkJcaxeRAG//e2Ul0+RhRo/kvsAUSYIPJf4jkAMYDnAHgZ+Q8vCyD/Q95Y7MCBA6qurlZ2dnateHZ2toqKihzjZ8+erczMzJpbu3btQr0koMEEmv8SewCxhecAeBnPAfAy8h9eEvbu3DNnzlRpaWnNbffu3eFeEtCg2APwMvIfXscegJeR/4hWIf84d8uWLZWQkKDi4uJa8eLiYuXk5DjGp6SkKCUlJdTLAMIi0PyX2AOILTwHwMt4DoCXkf/wkpBfiU5OTlafPn20fPnympjP59Py5cs1YMCAUB8OiCjkP7yOPQAvI//hZeQ/vKRevid6xowZGjt2rM4991z169dPc+bM0eHDhzV+/Pj6OBwQUch/eB17AF5G/sPLyH94Rb0U0VdffbX279+vu+66S0VFRerVq5eWLVvmaDQAxCLyH17HHoCXkf/wMvIfXlEv3xMdjLKyMmVmZmqwLqO1PcKiylRqpRartLRUTZo0afDjswcQTuQ/vI49AC8j/+FlgeR/2LtzAwAAAAAQLSiiAQAAAABwiSIaAAAAAACXKKIBAAAAAHCJIhoAAAAAAJcoogEAAAAAcIkiGgAAAAAAlyiiAQAAAABwiSIaAAAAAACXKKIBAAAAAHCJIhoAAAAAAJcoogEAAAAAcCkx3AsAAAD179vL+1vj7xY8bY3//ItBjljxgLKQrgn4voSmmdb4tt+eYY2bjCpHbPbAl61jr0r/lzW+r/pba/z8t6Y5Yp3/5DyeJMWv3miNA4hNXIkGAAAAAMAlimgAAAAAAFyiiAYAAAAAwCWKaAAAAAAAXKKIBgAAAADAJbpzAwAQQwLtwu3P8x3eccQ6PXKjdWzn6WsDmhvwZ9ek7tb49ivnWuPxinPEfDLWsT4/x2yZkGqNfzLcuWeWD2psHfvwz6+xxuPe/4efowKx79M551njn/34KUfswkkTrGMbL1oX0jWFCleiAQAAAABwiSIaAAAAAACXKKIBAAAAAHCJIhoAAAAAAJdoLBblElvnOGK7xnW0ju30g8+t8Zc7v2aNLz/ibJ4x8f3rrGPbvmxPpdTF661xwJ+EM7ta41unNLXGP/2RszlFQpz9/cFqY28r42/8tL3nOmIf3d3bOpZcR6QItIFYIAaet8UaL663I8Jrzh65LdxLOKmhqd9a44fnL7LGn+lqf00GxJKELvY8X3XFQ9Z4tXHWGIfaJljH2lv5hR9XogEAAAAAcIkiGgAAAAAAlyiiAQAAAABwiSIaAAAAAACXKKIBAAAAAHCJ7twRJj4tzRr/7I+drPFf93zLERvfxN5t2yd7Z2J7VBqS+o0jtnWovevrbd0HWOM7PmprjVft/srPUeEVVRf3scbv+aOz27YknZNsf8/Plr8+Ux3QWvyNfzBnnSO2ec571rHTKidb4ylLPwhoLYDNt5f3t8brsxO3zfMd3rHGL7x8gjXeeJFzDwEn8+H73azxMVXJ1vjWtXnOoAnsmE9d9Yw1PqjRUddz/CjtoDV+97TzrfGcOe+7nhtRLt7ZdTq+UYp1qO9be/f3SPfp3RnWeOsE9721W6/4tzXur04JN65EAwAAAADgEkU0AAAAAAAuUUQDAAAAAOASRTQAAAAAAC5RRAMAAAAA4BLduRtAXJKzo2R5fk/r2Fvm/tkaz09dGcARG/69EVsXY0n6zd/sHZg/HpLpiFWXlIZ0TYgcR3/Q1xF75MnHrWO7J0f2f0v+1pf7n59a4yVrmlnj1QftnVwBm4buwh0of+u7UHTtRmA63bLGGq/wM76jioI+5sNP/dAaH/TeoqDn9kX2UxoaQPEk57cr/OJG+zfpLBt5tjVe9cXukK6prhI7nmaNv3F+gZ9H2Ltzd17ifG7otm1DHVcVHlyJBgAAAADAJYpoAAAAAABcoogGAAAAAMAlimgAAAAAAFyiiAYAAAAAwCV6BoZQQpeO1vjWX7dwxLaNeqK+l+Nw34Fe1viOb1q5nuO/275qjbdNTLXG7271gTV+3k+nOmKtHn/f9ToQmYqmn2+NPz1lriMWaBdun3zW+Jlv3+CItV7s7IgvSeXN7O8bJl6+3xp/42xnt/zGcfa5nzvtLWt8wJjJ1niLP9g70MLbPn3kPD/3bGzIZYRM3q1brfHi4JseAwGL73mGNb791kZBz11hKq3xpp9XBz03osPBsQOs8Rf+4/eOWF5ignXs4o5DrfGECOnOvX1ijjXePtHehdufJtuSHDFTVVWnNYULV6IBAAAAAHCJIhoAAAAAAJcoogEAAAAAcIkiGgAAAAAAlwJuLPbOO+/owQcfVGFhofbu3atFixZp9OjRNfcbYzRr1iw9++yzKikp0cCBA/Xkk0+qS5cuoVx3RGrzP8XW+OK2Cx0xe4ukwPX74OeOWOLrTa1js1/aYo1Xlxx0fbxFm862xqc02+F6Dkn65URng7K/Pe5swBZpyP9j4jMyrPFOl9vzoE+K+7k3H7U3lpjxq4nWeOe3Cl3Pnebvjmft4b5/udER2zzoT66PJ0n/6m1vKhP52e5E/te/z65+qsGP+fMvBlnjz3d4J+i5/c1x4eUTHLHGi9YFfbz6xh6IQPHOBk077+1nHfr6dQ9a4+39NEcNxA1fjLDGG/+/yM9rt8j/Y+LT7K8mzp28wRo/Pcn5Iqjnup9Zx7ZZ8VHdF9YA4tscCWj8QZ99fJs3Djhi0daCL+Ar0YcPH1bPnj1VUFBgvf+BBx7QY489pqeeekrr1q1TWlqahg8frvLy8qAXC4Qb+Q8vI//hdewBeBn5D5wQ8JXoESNGaMQI+7ttxhjNmTNHv/nNb3TZZZdJkp5//nllZ2frlVde0TXXXBPcaoEwI//hZeQ/vI49AC8j/4ETQvo30Tt37lRRUZHy8/NrYpmZmerfv7/WrLF/J2pFRYXKyspq3YBoVJf8l9gDiA3kP7yOPQAvI//hNSEtoouKiiRJ2dnZteLZ2dk1933f7NmzlZmZWXNr165dKJcENJi65L/EHkBsIP/hdewBeBn5D68Je3fumTNnqrS0tOa2e/fucC8JaFDsAXgZ+Q+vYw/Ay8h/RKuA/yb6ZHJyciRJxcXFat26dU28uLhYvXr1sj4mJSVFKSkBtO5tQHFJydb49sd7WuNL2tq7qsYrzhq1ue/AWfa551xkjefM8/8Rme/z1/UukPNc0sx+jklxzu6YklRpXC0tJtQl/6XI3gP+7LylhzX+cafHXc+xt9resfGOYc6O85KUtMN9F25/Epo1s8Z3TjnDGv/gwt9bovb94k+fsz63xg8FNEvk81L+h0L2miYNfsxOLzm7zUvh6Qi+Z5DzebHzogZfRkixB8Lj33/r5IhtPsffc1HwXbg/qTxqjX99v70DdSOtD/qY0cBL+b//Gvs31SzJtTdcqzDObx2Jfy8zpGuqDwmd8xyxJec/4We0fW/1fetma7zrluBf04VbSK9E5+XlKScnR8uXL6+JlZWVad26dRowYEAoDwVEHPIfXkb+w+vYA/Ay8h9eE/CV6G+++Uaffvppzc87d+7Uxo0b1bx5c7Vv317Tpk3Tvffeqy5duigvL0933nmncnNza32PHBCtyH94GfkPr2MPwMvIf+CEgIvoDz/8UEOGDKn5ecaMGZKksWPHav78+br11lt1+PBh3XDDDSopKdEFF1ygZcuWqVGjRqFbNRAm5D+8jPyH17EH4GXkP3BCwEX04MGDZYz/P3SNi4vT3XffrbvvvjuohQGRiPyHl5H/8Dr2ALyM/AdOCHt3bgAAAAAAokVIu3PHmvJ8exfubaPsnel8fmdyvlcxdle+ZZz074u/tcabV7jvwu1PQrfO1vjWW5pa49tGOM/T3zn668Lt8/OIPzxxqSPWSu/7mR2R5uyLPwl6jkM+e0f36h32btYJLVtY4752OY7Y51fbOx8/duWfrPGhqW9Z44F24rY5dJSPscHp+Q7vhGQeW8ftztPXWsd2lj0+fHova/zTR85zxMLRyRs4mbXnLHDE/L8eC95DRcOs8cbLN1nj9bkW1K94Px9DP/+mDwOa54YvnTnT+veR/5r3kxuzHbFOiYF1uE/fEl2d1wPBlWgAAAAAAFyiiAYAAAAAwCWKaAAAAAAAXKKIBgAAAADAJYpoAAAAAABcojv3STS6ZU+9zX1O5pfW+FOzLwl67hdGF1jjmfH2ToAdk5KCPqY/l2z6sTXe6ol19XZMRIfcBHtL9/1/62aNv3D2PGu8c1JkdH78xldhjR+Zm2uNp+qr+lwOIoity7W0MSRz++vEHemidd3AM+1WWuPdfzPZGs+7I/hvV0F4HLj2HGt8SWv76+xtlfbXATvnnO6Ipfv5toRIEt/miOuxe6vt3y7Ubsl+a7y6TiuKLFyJBgAAAADAJYpoAAAAAABcoogGAAAAAMAlimgAAAAAAFyisdhJ3Hnaq/U2983NttnjV9vjgYj3896IT/XXQMyfkr+3tsZTfTsbeCUIpa2v2Jt/afrfXc+RHm9vCLauz4t+HhEZDcT86f3WFGu86yvrG3glCJdvL+9vjX929VNBz93ppRut8c5R0JwGqE/dFkx0xO4d9ZJ1bMfkfdZ4n+SEoNcxJH+jNb7rjqCnRpgc6B9Y+6sfvevMRUnq/NfI/n86oUtHa3zxgCct0UbWsRe/cIs1nrc1dhvrcSUaAAAAAACXKKIBAAAAAHCJIhoAAAAAAJcoogEAAAAAcIkiGgAAAAAAl+jOfRI3PjnZGv9w2qMNvJLIN3TTldZ42z9utsYD63eISJP70PvWeO94e4fqBRN/74h1TUoO6Zq+q7DCHn/3267W+LRmnwR9zNTPIrt7OOrfnkFx9TZ35+kN392VruKIBp3+w5lL8/6jg3VswpmXWOPz35hnjTeLt3citrk9+01r/EZd4HoOhE98Wpoj9vaIh/2MbmyNLrrQ1s1aWrXZzzeaNLCH3x9mjZ/Vdbc13jXJff5XNTbWeGKbXPv4r/e4njtScSUaAAAAAACXKKIBAAAAAHCJIhoAAAAAAJcoogEAAAAAcIkiGgAAAAAAl+jOfRK5D9o7EP/owb7W+L6J51vjZecfccR8R+y/+ovP2mqNb53TwxpPPOJzxA61sc8dX2XvnLd21uPWeCAOvNPaGk8t2Rn03IgeuQ/Y98yv//4LR+yrYU2tYyt6f2ON+3z29/w6/CHBEUspts+xb0Bza3zab9135/7b4WbW+GnPfmqN04k+9nx7eX9rPFq7WX/6yHl+7tlYb8cEwsGXZv8WhVBcUcp/1/7tFJ21IQSzo74d+PHZjlj7xHcDmqO7n28d6d40Ml4LTxz5dL3NvWy081tYJGnXD5ta4w9f/WNHzBTav9EnUnElGgAAAAAAlyiiAQAAAABwiSIaAAAAAACXKKIBAAAAAHCJIhoAAAAAAJfozh1CrZ6wdyZu9YT7Ob7yE88IoDNrqp/4gRsGWOM+OTt8B+q0vxZZ43QmhiT5Nm5xxHI31t/x4nKyrfFLp3wc9Nx3LLzOGj+teE3QcyM65N1q/xaFUOg8vf66cNdnV3F/ct+xfysEokNi2zbWeNVXXzfwSkLjixEZ1nhmfKOg5zb/tndmRnTIemWbI9btzEnWsaMu/sAaX/xBb2s8+V/ObxFp/HWcdezRpvb1Ndllf61uLJdDTZx97rwbt1vjL5z2lv2gFl1fmmiNN//Yfsysdf+yxs025+872nAlGgAAAAAAlyiiAQAAAABwiSIaAAAAAACXKKIBAAAAAHCJxmIe0vonu4KeY391hTVevePzoOcGQuXLn3ayxl9puTSgefZWH3HE2i8rr9OaEDue7/BO0HP8/ItBfu4pC3ru7DVNrPHnOzwd9Nz+dHrpRmu886L6a5SG0No36XxH7HfTn7WO/X3n7vW9HNfi09Icse3397COffXS3/uZJfimYO3+ThO9aFZ98KAj1ukWe8NQf60lu2p9CFcUeuv725tLKoDGYs032RuINZ9n/13FcoNhrkQDAAAAAOASRTQAAAAAAC5RRAMAAAAA4BJFNAAAAAAALlFEAwAAAADgEt25Y1BCt87W+Mz2fw167iEv3mKNd5S9Kx8QDlXpoZnn8n9c74i1fHdDaCaHp7239kxrPPdye4ffPYPsHVE/u/qpkK3JDX9dxTtPpwt3tPvJTW84Yv0b2bvFf/rwedb46b//0hqv+npP3Rf2fw6OHWCN/+g/Vjhif2vxhJ9Zgu/C3efhKdZ47puF1jg9u9HQEpo1s8bvyl8U9Nwt1zu7mEuSL+iZow9XogEAAAAAcIkiGgAAAAAAlyiiAQAAAABwiSIaAAAAAACXAiqiZ8+erb59+yojI0OtWrXS6NGjtX379lpjysvLNWnSJLVo0ULp6ekaM2aMiouLQ7poIFzYA/Ay8h9eRv7D69gDwAkBdedetWqVJk2apL59+6qqqkp33HGHhg0bpi1btigtLU2SNH36dL322mtauHChMjMzNXnyZF1xxRV677336uUE4LRtYktr/NyU6qDn7vjK4aDniGbsgciTkN3KEbvtmv8Nydwt7m8cknlihdfy/9NH7B2IpY1Bz+23q/bVQU8dMrZO3MUD7N2avcBr+S9JjePs3ay3XV1gjW+9otIaP+QLviv2OcnrrfGUOOdL2UA7Bfv8POL0v9/oiHV73E8X7oqKAI8afby4B6JRaX5Xa/xnGcsDmucH2y5zxOK37KjTmmJRQEX0smXLav08f/58tWrVSoWFhRo0aJBKS0v1xz/+US+++KIuvvhiSdK8efN0xhlnaO3atTrvPH8vSIDowB6Al5H/8DLyH17HHgBOCOpvoktLSyVJzZs3lyQVFhaqsrJS+fn5NWNOP/10tW/fXmvW2L9HuKKiQmVlZbVuQLRgD8DLyH94WSjyX2IPIHrxHAAvq3MR7fP5NG3aNA0cOFA9evSQJBUVFSk5OVlNmzatNTY7O1tFRUXWeWbPnq3MzMyaW7t27eq6JKBBsQfgZeQ/vCxU+S+xBxCdeA6A19W5iJ40aZI2bdqkBQsWBLWAmTNnqrS0tOa2e/fuoOYDGgp7AF5G/sPLQpX/EnsA0YnnAHhdQH8TfdzkyZO1ZMkSvfPOO2rbtm1NPCcnR0ePHlVJSUmtd6GKi4uVk5NjnSslJUUpKSl1WQYQNuwBeBn5Dy8LZf5L7AFEH54DgACLaGOMpkyZokWLFmnlypXKy8urdX+fPn2UlJSk5cuXa8yYMZKk7du368svv9SAAQNCt2qcXLOj1nB8gB88mFfm/EhN4hf7rGOrApo5erEHIs/W+515el3G6yGZO/GTrx2x4HvcRy/y31t2PnCGI9ZY68KwkshA/p/aGUlJfu4xIZg9wR6Nc7628Rn7/9Sbj9pfrYx9dLo13nXO+45YKM4kWrEHokPRwLiQzHPgZefrq1Y+PilwXEBF9KRJk/Tiiy9q8eLFysjIqPn7hszMTKWmpiozM1O/+MUvNGPGDDVv3lxNmjTRlClTNGDAADryISawB+Bl5D+8jPyH17EHgBMCKqKffPJJSdLgwYNrxefNm6dx48ZJkh555BHFx8drzJgxqqio0PDhw/XEE0+EZLFAuLEH4GXkP7yM/IfXsQeAEwL+OPepNGrUSAUFBSooKKjzooBIxR6Al5H/8DLyH17HHgBOCOp7ogEAAAAA8JI6dedGZHvi/BescZ98Ac2z8t/dHLGqvf6/6xKoT3FJydb47wb+bwOvBIhsP/9ikDX+3tozrfHO09da415uIuZFT3442BG7KX+zdWxKnL8GYg2v2jhf29xS1N86duNd51jjOa85G4gB0eqGocsDGr+tssIaz331S0fMK42E3eBKNAAAAAAALlFEAwAAAADgEkU0AAAAAAAuUUQDAAAAAOASRTQAAAAAAC7RnTvKmYG9HLGhqYXWsYH15gYiy/Ynelrjl6etcT2Hvw71/T74uTWeW/6V67kRe/x1rdbVDbsOSer00o2ux/pbd2f5OR9AUpdxztcOF06cYR0bP+KANb6291+CXsdfv2lljf9m9eX2tRxyvpTtdtcW69iUsg/qvjAgEvU7yxGa1vxPfgYnWKMTtl1njaft/ryuq/IErkQDAAAAAOASRTQAAAAAAC5RRAMAAAAA4BJFNAAAAAAALlFEAwAAAADgEt25o9w37RoFPcch31FrfM2Wzo5YV9HZEuEx4/y/Bz3H2F3DrfGc0VutcTraw2Z4bq8GPyadtREOrZ54337HE/bwKPWpt7V01Yeux1bX2yqAyJJQesQRe6bE+fpdkiY23WmNH/5bjjWeJrpznwxXogEAAAAAcIkiGgAAAAAAlyiiAQAAAABwiSIaAAAAAACXKKIBAAAAAHCJ7txRLvGIs3/ww/8+3Tq2ZeIha/y/377MGu86aV3dFwaE2Jy/j7DGb/xxgSN24T+uto5tcVOFn9n/XddlAQAAhEX19k8dsSXdm1nHLpE93kp+uvDjpLgSDQAAAACASxTRAAAAAAC4RBENAAAAAIBLFNEAAAAAALhEY7Eol7p4vSP29uI0P6Pt8S6igRgiX+fpa63xUdP7OGKZcjbakKSqkK4IAAAAXsSVaAAAAAAAXKKIBgAAAADAJYpoAAAAAABcoogGAAAAAMAlimgAAAAAAFyiiAYAAAAAwCWKaAAAAAAAXKKIBgAAAADAJYpoAAAAAABcoogGAAAAAMClxHAv4PuMMZKkKlVKJsyLgSdVqVLSiVxsaOwBhBP5D69jD8DLyH94WSD5H3FF9KFDhyRJq7U0zCuB1x06dEiZmZlhOa7EHkB4kf/wOvYAvIz8h5e5yf84E663mvzw+Xzas2ePMjIydOjQIbVr1067d+9WkyZNwr20elFWVhbz5yhF13kaY3To0CHl5uYqPr7h/+KBPRB7oukcyf+GFU25EYxoOs9I2QPGGLVv3z4qfmfBiKbcqKtoOsdIyX+eA2JHNJ1jIPkfcVei4+Pj1bZtW0lSXFycJKlJkyYR/0sPlhfOUYqe8wzHu6/HsQdiV7ScI/nf8LxwjlL0nGck7IGysjJJ0fM7C5YXzjNazjES8l/iOSDWRMs5us1/GosBAAAAAOASRTQAAAAAAC5FdBGdkpKiWbNmKSUlJdxLqTdeOEfJO+cZal74vXGO8McLvzcvnKPknfMMJa/8zrxwnl44x/rghd8b5xi9Iq6xGAAAAAAAkSqir0QDAAAAABBJKKIBAAAAAHCJIhoAAAAAAJcoogEAAAAAcCmii+iCggKddtppatSokfr376/169eHe0l19s477+jSSy9Vbm6u4uLi9Morr9S63xiju+66S61bt1Zqaqry8/O1Y8eO8Cy2jmbPnq2+ffsqIyNDrVq10ujRo7V9+/ZaY8rLyzVp0iS1aNFC6enpGjNmjIqLi8O04sgWS/kvxf4eIP9DL5b2QKznv8QeCDXyn/z3sljKf4k9cFws7YGILaJfeuklzZgxQ7NmzdJHH32knj17avjw4dq3b1+4l1Ynhw8fVs+ePVVQUGC9/4EHHtBjjz2mp556SuvWrVNaWpqGDx+u8vLyBl5p3a1atUqTJk3S2rVr9eabb6qyslLDhg3T4cOHa8ZMnz5dr776qhYuXKhVq1Zpz549uuKKK8K46sgUa/kvxf4eIP9DK9b2QKznv8QeCCXyn/z3sljLf4k9cFxM7QETofr162cmTZpU83N1dbXJzc01s2fPDuOqQkOSWbRoUc3PPp/P5OTkmAcffLAmVlJSYlJSUsxf/vKXMKwwNPbt22ckmVWrVhljjp1TUlKSWbhwYc2YrVu3GklmzZo14VpmRIrl/DfGG3uA/A9OLO8BL+S/MeyBYJD/5L+XxXL+G8MeiJU9EJFXoo8eParCwkLl5+fXxOLj45Wfn681a9aEcWX1Y+fOnSoqKqp1vpmZmerfv39Un29paakkqXnz5pKkwsJCVVZW1jrP008/Xe3bt4/q8ww1r+W/FJt7gPyvO6/tgVjMf4k9UFfkP/nvZV7Lf4k9EK17ICKL6AMHDqi6ulrZ2dm14tnZ2SoqKgrTqurP8XOKpfP1+XyaNm2aBg4cqB49ekg6dp7Jyclq2rRprbHRfJ71wWv5L8XeHiD/g+O1PRBr+S+xB4JB/qvm52g9X/K/7ryW/xJ7IFrPMzHcC0BsmjRpkjZt2qTVq1eHeylAgyP/4XXsAXgZ+Q+v88IeiMgr0S1btlRCQoKjW1txcbFycnLCtKr6c/ycYuV8J0+erCVLlmjFihVq27ZtTTwnJ0dHjx5VSUlJrfHRep71xWv5L8XWHiD/g+e1PRBL+S+xB4JF/qvm52g8X/I/OF7Lf4k9EK3nGZFFdHJysvr06aPly5fXxHw+n5YvX64BAwaEcWX1Iy8vTzk5ObXOt6ysTOvWrYuq8zXGaPLkyVq0aJHefvtt5eXl1bq/T58+SkpKqnWe27dv15dffhlV51nfvJb/UmzsAfI/dLy2B2Ih/yX2QKiQ/+S/l3kt/yX2QNTugbC2NTuJBQsWmJSUFDN//nyzZcsWc8MNN5imTZuaoqKicC+tTg4dOmQ2bNhgNmzYYCSZhx9+2GzYsMF88cUXxhhj7r//ftO0aVOzePFi889//tNcdtllJi8vzxw5ciTMK3fvpptuMpmZmWblypVm7969Nbdvv/22ZsyNN95o2rdvb95++23z4YcfmgEDBpgBAwaEcdWRKdby35jY3wPkf2jF2h6I9fw3hj0QSuQ/+e9lsZb/xrAHjoulPRCxRbQxxsydO9e0b9/eJCcnm379+pm1a9eGe0l1tmLFCiPJcRs7dqwx5lh7+zvvvNNkZ2eblJQUM3ToULN9+/bwLjpAtvOTZObNm1cz5siRI2bixImmWbNmpnHjxubyyy83e/fuDd+iI1gs5b8xsb8HyP/Qi6U9EOv5bwx7INTIf/Lfy2Ip/41hDxwXS3sgzhhjQnNNGwAAAACA2BaRfxMNAAAAAEAkoogGAAAAAMAlimgAAAAAAFyiiAYAAAAAwCWKaAAAAAAAXKKIBgAAAADAJYpoAAAAAABcoogGAAAAAMAlimgAAAAAAFyiiAYAAAAAwCWKaAAAAAAAXKKIBgAAAADAJYpoAAAAAABcoogGAAAAAMAlimgAAAAAAFyiiAYAAAAAwCWKaAAAAAAAXKKIBgAAAADAJYpoAAAAAABcoogGAAAAAMAlimgAAAAAAFyiiAYAAAAAwCWKaAAAAAAAXKKIBgAAAADAJYpoAAAAAABcoogGAAAAAMAlimgAAAAAAFyiiAYAAAAAwCWKaAAAAAAAXKKIBgAAAADAJYpoAAAAAABcoogGAAAAAMAlimgAAAAAAFyiiAYAAAAAwCWKaAAAAAAAXKKIBgAAAADAJYpoAAAAAABcoogGAAAAAMAlimgAAAAAAFyiiAYAAAAAwCWKaAAAAAAAXKKIBgAAAADAJYpoAAAAAABcoogGAAAAAMAlimgAAAAAAFyiiAYAAAAAwCWKaAAAAAAAXKKIBgAAAADAJYpoAAAAAABcoogGAAAAAMAlimgAAAAAAFyiiAYAAAAAwCWKaAAAAAAAXKKIBgAAAADAJYpoAAAAAABcoogGAAAAAMAlimgAAAAAAFyiiAYAAAAAwCWKaAAAAAAAXKKIBgAAAADAJYpoAAAAAABcoogGAAAAAMAlimgAAAAAAFyiiAYAAAAAwCWKaAAAAAAAXKKIBgAAAADAJYpoAAAAAABcoogGAAAAAMAlimgAAAAAAFyiiAYAAAAAwCWKaAAAAAAAXKKIBgAAAADAJYpoAAAAAABcoogGAAAAAMAlimgAAAAAAFyiiAYAAAAAwCWKaAAAAAAAXKKIBgAAAADAJYpoAAAAAABcoogGAAAAAMAlimgAAAAAAFyiiAYAAAAAwCWKaAAAAAAAXKKIBgAAAADAJYpoAAAAAABcoogGAAAAAMAlimgAAAAAAFyiiAYAAAAAwCWKaAAAAAAAXKKIBgAAAADAJYroCHLaaadp3LhxNT+vXLlScXFxWrlyZciOERcXp//6r/8K2XxAKLEH4GXkP7yOPQAvI/+jC0X0/5k/f77i4uJqbo0aNVLXrl01efJkFRcXh3t5AVm6dGlUbZDHH39cZ5xxhlJSUtSmTRvNmDFDhw8fDveyPIc9ED5//etfdd5556lp06Zq0aKFLrroIr322mvhXpankP/h8d3f+fdvl1xySbiX5ynsgfBYv369Jk6cqD59+igpKUlxcXHhXpInkf/hE62vgRLDvYBIc/fddysvL0/l5eVavXq1nnzySS1dulSbNm1S48aNG3QtgwYN0pEjR5ScnBzQ45YuXaqCggLrBjpy5IgSEyPnn/22227TAw88oCuvvFJTp07Vli1bNHfuXG3evFlvvPFGuJfnSeyBhjV37lzdfPPN+uEPf6j7779f5eXlmj9/vkaNGqWXX35ZV1xxRbiX6Cnkf8P685//7Ih9+OGHevTRRzVs2LAwrAjsgYa1dOlS/eEPf9DZZ5+tjh076pNPPgn3kjyN/G9Y0fwaKHJ+ixFixIgROvfccyVJv/zlL9WiRQs9/PDDWrx4sa699lrrYw4fPqy0tLSQryU+Pl6NGjUK6Zyhni8Ye/fu1cMPP6yf/exnev7552viXbt21ZQpU/Tqq6/q0ksvDeMKvYk90LDmzp2rvn376tVXX625AnH99derTZs2eu655yL6CSQWkf8N66c//akjdvwjjP5+36hf7IGGddNNN+m2225TamqqJk+eTBEdZuR/w4rm10B8nPsULr74YknSzp07JUnjxo1Tenq6PvvsM40cOVIZGRm67rrrJEk+n09z5sxR9+7d1ahRI2VnZ2vChAk6ePBgrTmNMbr33nvVtm1bNW7cWEOGDNHmzZsdx/b3txDr1q3TyJEj1axZM6Wlpenss8/Wo48+WrO+goICSbU/Jnec7W8hNmzYoBEjRqhJkyZKT0/X0KFDtXbt2lpjjn/M5b333tOMGTOUlZWltLQ0XX755dq/f3+tsaWlpdq2bZtKS0tP+rtds2aNqqqqdM0119SKH/95wYIFJ308GgZ74Jj62AOSVFZWplatWtVa4/F1pKamnvLxqF/k/zH1lf/fV1FRoZdfflkXXXSR2rZtG/DjEXrsgWPqaw9kZ2fzf30EI/+P4TWQE1eiT+Gzzz6TJLVo0aImVlVVpeHDh+uCCy7QQw89VPPxjgkTJmj+/PkaP368br75Zu3cuVOPP/64NmzYoPfee09JSUmSpLvuukv33nuvRo4cqZEjR+qjjz7SsGHDdPTo0VOu580339SoUaPUunVrTZ06VTk5Odq6dauWLFmiqVOnasKECdqzZ4/efPNN68fkvm/z5s268MIL1aRJE916661KSkrS008/rcGDB2vVqlXq379/rfFTpkxRs2bNNGvWLO3atUtz5szR5MmT9dJLL9WMWbRokcaPH6958+bVapDwfRUVFZLk2CTHf5+FhYWnXD/qH3ug/vaAJA0ePFj/+7//q7lz5+rSSy9VeXm55s6dq9LSUk2dOvWU60f9Iv/rN/+/b+nSpSopKal5UYrwYw807B5AZCH/eQ3kl4Exxph58+YZSeatt94y+/fvN7t37zYLFiwwLVq0MKmpqearr74yxhgzduxYI8ncfvvttR7/7rvvGknmhRdeqBVftmxZrfi+fftMcnKy+eEPf2h8Pl/NuDvuuMNIMmPHjq2JrVixwkgyK1asMMYYU1VVZfLy8kyHDh3MwYMHax3nu3NNmjTJ+PunlWRmzZpV8/Po0aNNcnKy+eyzz2pie/bsMRkZGWbQoEGO309+fn6tY02fPt0kJCSYkpISx9h58+ZZ13BcYWGhkWTuueeeWvHjv7P09PSTPh6hxR5o+D1gjDHFxcVm6NChRlLNrWXLlub9998/5WMROuR/ePL/+8aMGWNSUlIc54f6xx4I/x442bpRv8h/XgMFio9zf09+fr6ysrLUrl07XXPNNUpPT9eiRYvUpk2bWuNuuummWj8vXLhQmZmZuuSSS3TgwIGaW58+fZSenq4VK1ZIkt566y0dPXpUU6ZMqfXRhWnTpp1ybRs2bNDOnTs1bdo0NW3atNZ9denmWF1drb///e8aPXq0OnbsWBNv3bq1fvKTn2j16tUqKyur9Zgbbrih1rEuvPBCVVdX64svvqiJjRs3TsaYU7771Lt3b/Xv31+/+93vNG/ePO3atUuvv/66JkyYoKSkJB05ciTgc0Lw2AMNtwekY5+86Natm8aOHauFCxfqT3/6k1q3bq0rrrhCn376acDnhOCQ/w2b/99VVlam1157TSNHjnScHxoOeyB8ewDhR/7zGsgtPs79PQUFBeratasSExOVnZ2tbt26KT6+9nsNiYmJjr/V2rFjh0pLS9WqVSvrvPv27ZOkmiTr0qVLrfuzsrLUrFmzk67t+EdKevTo4f6ETmL//v369ttv1a1bN8d9Z5xxhnw+n3bv3q3u3bvXxNu3b19r3PE1f//vPdx6+eWXdfXVV+v666+XJCUkJGjGjBlatWqVtm/fXqc5ERz2wDENtQeuuuoqJSYm6tVXX62JXXbZZerSpYv+8z//s9ZHpFD/yP9jGir/v+vll19WeXk5H+UOM/bAMeHYAwg/8v8YXgOdGkX09/Tr16+mK58/KSkpjg3l8/nUqlUrvfDCC9bHZGVlhWyN4ZSQkGCNG2PqNF+bNm20evVq7dixQ0VFRerSpYtycnKUm5urrl27BrNU1BF74ORCuQc+//xzLVu2TM8880ytePPmzXXBBRfovffeq9MaUXfk/8mF+jngu1544QVlZmZq1KhRQc+FumMPnFx97gGEH/l/crwGOoEiOkQ6deqkt956SwMHDjxpN7kOHTpIOvaO1Xc/OrF///5TvovTqVMnSdKmTZuUn5/vd5zbj3RkZWWpcePG1iu+27ZtU3x8vNq1a+dqrmB16dKl5l25LVu2aO/evXwMKsqwBwJXXFws6dhHqr6vsrJSVVVV9XZshBb5H5y9e/dqxYoVGjdunFJSUhrkmAgt9gC8jPwPXLS/BuJvokPkxz/+saqrq3XPPfc47quqqlJJSYmkY39rkZSUpLlz59Z612bOnDmnPEbv3r2Vl5enOXPm1Mx33HfnOv5ddd8f830JCQkaNmyYFi9erF27dtXEi4uL9eKLL+qCCy5QkyZNTrmu7wvm6018Pp9uvfVWNW7cWDfeeGPAj0f4sAdOcLsHOnfurPj4eL300ku11v/VV1/p3Xff1TnnnBPwsREe5P8JdXkOWLBggXw+Hx/ljmLsgROCeR2E6ET+n+CV10BciQ6Riy66SBMmTNDs2bO1ceNGDRs2TElJSdqxY4cWLlyoRx99VFdeeaWysrL061//WrNnz9aoUaM0cuRIbdiwQa+//rpatmx50mPEx8frySef1KWXXqpevXpp/Pjxat26tbZt26bNmzfrjTfekCT16dNHknTzzTdr+PDhSkhIcHwX83H33nuv3nzzTV1wwQWaOHGiEhMT9fTTT6uiokIPPPBAnX4XgbS2nzp1qsrLy9WrVy9VVlbqxRdf1Pr16/Xcc885/u4CkY09cILbPZCVlaXrr79ef/jDHzR06FBdccUVOnTokJ544gkdOXJEM2fOrNPx0fDI/xPq8vU+L7zwgnJzczV48OA6HRPhxx44IZA98MUXX9R8FdGHH35Ysybp2FXLn/3sZ3VaAxoW+X+CZ14DNWQr8Eh2vB37Bx98cNJxY8eONWlpaX7vf+aZZ0yfPn1MamqqycjIMGeddZa59dZbzZ49e2rGVFdXm9/+9remdevWJjU11QwePNhs2rTJdOjQ4aSt7Y9bvXq1ueSSS0xGRoZJS0szZ599tpk7d27N/VVVVWbKlCkmKyvLxMXF1Wpzr++1tjfGmI8++sgMHz7cpKenm8aNG5shQ4Y4Wsv7+/3Y1hhIa/t58+aZnj17mrS0NJORkWGGDh1q3n777VM+DqHHHgjPHqisrDRz5841vXr1Munp6SY9Pd0MGTKEfdDAyP/w5L8xxmzbts1IMjNmzHA1HvWDPRCePXD88bbbRRdddMrHIzTIf14DBSrOGDohAAAAAADgBn8TDQAAAACASxTRAAAAAAC4RBENAAAAAIBLFNEAAAAAALhEEQ0AAAAAgEsU0QAAAAAAuJRYXxMXFBTowQcfVFFRkXr27Km5c+eqX79+p3ycz+fTnj17lJGRobi4uPpaHuCXMUaHDh1Sbm6u4uPr9j5TXfNfYg8gvEKR/xLPAYhePAfAy8h/eFlA+V8fXz69YMECk5ycbP70pz+ZzZs3m1/96lemadOmpri4+JSP3b17t98vnefGrSFvu3fvbvD8Zw9wi5RbXfM/2D1A/nOLlBvPAdy8fCP/uXn55ib/44wxRiHWv39/9e3bV48//rikY+8qtWvXTlOmTNHtt99+0seWlpaqadOmukAjlaikUC8NOKUqVWq1lqqkpESZmZkBPz6Y/JfYAwivYPNf4jkA0Y3nAHgZ+Q8vCyT/Q/5x7qNHj6qwsFAzZ86sicXHxys/P19r1qxxjK+oqFBFRUXNz4cOHfq/hSUpMY7NgzD4v7eV6vIxokDzX2IPIMIEkf8SzwGIATwHwMvIf3hZAPkf8sZiBw4cUHV1tbKzs2vFs7OzVVRU5Bg/e/ZsZWZm1tzatWsX6iUBDSbQ/JfYA4gtPAfAy3gOgJeR//CSsHfnnjlzpkpLS2tuu3fvDveSgAbFHoCXkf/wOvYAvIz8R7QK+ce5W7ZsqYSEBBUXF9eKFxcXKycnxzE+JSVFKSkpoV4GEBaB5r/EHkBs4TkAXsZzALyM/IeXhPxKdHJysvr06aPly5fXxHw+n5YvX64BAwaE+nBARCH/4XXsAXgZ+Q8vI//hJfXyPdEzZszQ2LFjde6556pfv36aM2eODh8+rPHjx9fH4YCIQv7D69gD8DLyH15G/sMr6qWIvvrqq7V//37dddddKioqUq9evbRs2TJHowEgFpH/8Dr2ALyM/IeXkf/winr5nuhglJWVKTMzU4N1Ga3tERZVplIrtVilpaVq0qRJgx+fPYBwIv/hdewBeBn5Dy8LJP/D3p0bAAAAAIBoQRENAAAAAIBLFNEAAAAAALhEEQ0AAAAAgEsU0QAAAAAAuEQRDQAAAACASxTRAAAAAAC4RBENAAAAAIBLFNEAAAAAALhEEQ0AAAAAgEsU0QAAAAAAuEQRDQAAAACAS4nhXgAAAAAQLomntbfGX3lvkes5Rp8/2hqv+mJ3XZYEIMJxJRoAAAAAAJcoogEAAAAAcIkiGgAAAAAAlyiiAQAAAABwiSIaAAAAAACX6M7dABK6dHTElq76f9axW49+a41PO+38kK6prj55op81HueLs8a7TF5Xn8sBFN/rTEds+7RU69gdlzxrjSfE2d9PvO9AN0ds1dn2uQEA0enLq9pa4z75ApijnTWe+xDdueFe2bXnWeP7Ly13PcdFHT+1xp9p9441PujjK63xw6/lOGK5z22yjq0uK3O5utjBlWgAAAAAAFyiiAYAAAAAwCWKaAAAAAAAXKKIBgAAAADAJRqLhVDCmV2t8U9+09gRqzTV1rEZ8fYmFvE9TrfGfZu2uVxdaMwcssQa/7KihTX+UZNm1rgXGxDAnbhE+39Le262N7X7w5RHHbE+yQkBHbPa2PfdpOYbHbGVA260jo1b84+AjgnY+HseUWWVNVy94/N6XI3T4TH9rfGFj/zeGr/kiVut8Tb3vx+yNQFu/euXA6zxp256POi526w6ZI2boGdG1Ot3liPU52n7a4bftiqwxv01uYu3XA/1N9bn59rp22e9ZJ/7LOf4EZt+ZR2bsOIjazyWcSUaAAAAAACXKKIBAAAAAHCJIhoAAAAAAJcoogEAAAAAcIkiGgAAAAAAl+jOHUJbp2Va459c9JTrOZrHJ1vjB3s1tcYzN7meOmAHJji7WI5v4q+D5W5rdHTLMfbhdOf2vMS2bazxb+fZ/1vacKa/3AusE3cg0uNSHLHk2fusY6svzbDGfYfsHVvhbVUX97HG7/vjk9Z4UXUTa/yxzvZvbqgvTad8aY23SnB+C4UknTlquzVeen/IlgS4VtEszho/N8X+jSmBSCg6aI3b++ojFn3+gL37+46fOv9f9/ctPfGy56i/65728YGMDWz81xc1so5tv8LP1DGMK9EAAAAAALhEEQ0AAAAAgEsU0QAAAAAAuEQRDQAAAACASxTRAAAAAAC4RHfuOvhq5vnW+KJLHvHzCPe/5rMW3myNd/6fta7nCBXfSHunSSBQtk7cT7//knVsaz9dfgOx/Iizq7Yk3fjaL+wPaHbUGt4x9A+O2OIur1nH/qjl5dY43bm9I7FNrjX++S9Oc8Reuv5h69juSfZvaDhiSqzxgqwsR6x6/377AgMU38PZ+fvnua+HZG6gPiVY9oUktRu5K+i5L/rHtdZ4s3/vDXpuRId//cLehXv5NQ9a45Um1RHzyWcdW1hhv745/rkp1nibd8qtcZt/dbd31l4781E/j7Csxbg+XMzjSjQAAAAAAC5RRAMAAAAA4BJFNAAAAAAALlFEAwAAAADgEkU0AAAAAAAu0Z27Ds677J/WePdk97/OeWXtrPFu931mjVe7njl0emV/7Xrsgeoj9jt89u6DiE0JzZpZ4z1edeZSoF24/+Wz59jAF37tiHV98ivr2C5f2Lvc73i+d0BrAWy+Ksi0xjf1fdwStXfh9ife33vemenOWIi6cx98oMoRG5Nm/9aGMp+9Q+zn87ta4y20pu4LA05hx6POb4SQpE1dnw167tRHm1rjvsOfBj03okPpUPvrkdYJzi7ckpQUl+CIVfrpcj2rYx9rvL3ed7e4k2iUc5417u/5JV5xjlgg3cBjHVeiAQAAAABwiSIaAAAAAACXKKIBAAAAAHCJIhoAAAAAAJcCbiz2zjvv6MEHH1RhYaH27t2rRYsWafTo0TX3G2M0a9YsPfvssyopKdHAgQP15JNPqkuXLqFcd4M4PKa/NT637Vw/j3A2DpCkg5aGK089epl1bNb+6Gy2cu22n1rjKbt2NexC6pmX8r8uvv75Gdb4q61sjZXsDvppIDbqN84GYpLU8TnnnnG2Qzrm6A/6WuOFQx7184hGjsjK8iTrSPOtn+Z6McRr+R+X5Kf51xtZ1vCH3f7H30yOiE/2rjKz9p1jjX90/VnWuPl0s59jupfQpIk1/kL3+ZaovSHgtTuussZb/CE6n9P88doeiArxztdeHbMPhGTqRd+0csRSd5VYx4ajAWxDI/+P8dMTTD7Zm+namoj5G/vJH861xtsvtl/3bPTqemewn/354sypm6xxf2t5qqSzI5ZcaG+gF3D+W9YY5+cXaz74ONDZG0TAV6IPHz6snj17qqCgwHr/Aw88oMcee0xPPfWU1q1bp7S0NA0fPlzl5XRzQ/Qj/+Fl5D+8jj0ALyP/gRMCvhI9YsQIjRgxwnqfMUZz5szRb37zG1122bErrc8//7yys7P1yiuv6JprrglutUCYkf/wMvIfXscegJeR/8AJIf2b6J07d6qoqEj5+fk1sczMTPXv319r1tg/0lVRUaGysrJaNyAa1SX/JfYAYgP5D69jD8DLyH94TUiL6KKiIklSdnZ2rXh2dnbNfd83e/ZsZWZm1tzatWsXyiUBDaYu+S+xBxAbyH94HXsAXkb+w2vC3p175syZKi0trbnt3r073EsCGhR7AF5G/sPr2APwMvIf0Srgv4k+mZycHElScXGxWrduXRMvLi5Wr169rI9JSUlRSkpKKJcRMl+PsPeaS4qzd+H2p8TS9M4kOLu1SlJCtrMTpCTpaKU1XH3wYEBrsYk7p7s1/rs2z1iizm7FkvRtpb1jcWT+y9aPuuS/FNl7IFDftvbXs9K9S2bfYo23eu79oOf+YpT9fcMm8fa8tpnwyq+s8U7Fa+u0plgRi/mf+GYLa3xxl1f9PMLP/+txzrx76F/2brWF5/h7bzv4Ltz+7P1pD2v8tMSVrufY8U/71aPO+rouS4pKsbgHosHO+/o5Yh+f/lhAc3xeaX+N9dxVwx0x3/ZtAc3tFV7K/2632bu/L1luf84YnVZiidr/r/90hO21t+QbYX991W3YRGvcZmm7d+xz+1nLw2succS6ln3o+ngnU5Xh/PaLOJ/9HAOruhpOSK9E5+XlKScnR8uXL6+JlZWVad26dRowYEAoDwVEHPIfXkb+w+vYA/Ay8h9eE/CV6G+++UaffnriO8J27typjRs3qnnz5mrfvr2mTZume++9V126dFFeXp7uvPNO5ebm1voeOSBakf/wMvIfXscegJeR/8AJARfRH374oYYMGVLz84wZMyRJY8eO1fz583Xrrbfq8OHDuuGGG1RSUqILLrhAy5YtU6NG7j8uCUQq8h9eRv7D69gD8DLyHzgh4CJ68ODBMsb/3z3GxcXp7rvv1t133x3UwoBIRP7Dy8h/eB17AF5G/gMnhL07NwAAAAAA0SKk3bljzQ97fhySefISnR9jWX/HXPvgO+zh9RX2rq9TNl3riB19394dsPm2Kmt8t7P5pCSpWQAdixP+x35MaYfrORD9EvzkaSDeveNha/yZiWda46/8Jt8Ry/jQ3hF42aX2uaVUa/QbU+GI5b3ijCH6xVs+bvinTv/rZ7Q9X/z52+HGjtjyCedbx8bpHwHNHYjEtm2s8WtuejPoueMqg9/7QF1s/tnjjpjlS1FOqrg63Rr3/ZNO3HCq2v2VNf7sdZdZ4w/fc9gRe/usl/zMbr++6fOT1duveML1WH9duP2NP/O/9jpi9kqiDoL/Mpew40o0AAAAAAAuUUQDAAAAAOASRTQAAAAAAC5RRAMAAAAA4BJFNAAAAAAALtGd+yQqTeS8x9Avxd7Gbl2fF53BPvW8GIt/97B3Zm3SwOtAeOU9YO8sfFH/Kx2xVWfZOx+nxiVb41ObfWqNj5izyRF7/Zse1rGdEgPrqjx66zWOWPK7GwKaA9Fhz429HbEW8e+HZO4fNv7GEbvor09Zx168Yaw1nvCy/RsQms1f43odVW3tc/y6+XbXc2w8au/N2m3OF/Zjup4ZOLm9M+wd7aXCoOe+ftmvrPEuWhf03PAO84H9W33Sf+CM/Uh9rWM/f2CANX7VJe9Z4/e02miJ2uuXeNlfqw/++GprPP2rz63xkLAtJco6dkdOlQgAAAAAQISjiAYAAAAAwCWKaAAAAAAAXKKIBgAAAADAJRqLncQXEzpa4//xrL1VSkq8PX5JE2fjoyGp5XVfWAS676oXrPFb2jobSklS17lHHbGE/aXWsVVf7K77wtCgfIcPW+Ppl33tiJ3168nWsQMu/ac1/ky7d6zxrkmNnDE/Tcj8WVmeZI2nTnL+F1kd0MyIFnG+hj1eelyKNb6+9wL7A5x9zyRJT93awfUxmya85nqsP1e9OsUa7/I1DZhQv5oMK7LGk+ISHLFKPw2Kpu+xNyfrOmOjNR5lfY4QAzream8WWejnuqfva2eW+uTvCc0+x9tnvWSN+2t+FhIxsLm4Eg0AAAAAgEsU0QAAAAAAuEQRDQAAAACASxTRAAAAAAC4RBENAAAAAIBLdOc+CbNhszW+/dzA5tnc5VJH7HftmgU0x+5hyfY7TnN2Q85tXmYd+mb3lwM6ZiB+lHbQHs9/1hrfOdjZnfztw12tYxedmVX3hSEimIoKR6zdf79vHfv1A/ZcH3DdJGv8h9NXOWK/aensiF8ncXGhmQcRL/sxZz72jrd3kG82co813j9rlzXetZGzq/C4JvY5AnVj5hchmcetlAPOTsiSFN/I2SVfknzlsfVNFKh/lcPsL7JePHOOfbxJdcT8dSdetuoca7xTxVp3iwPCZO8rZ1jj8frIGrWP9feaxj7edszWo7f6mSNAtqVEWcdurkQDAAAAAOASRTQAAAAAAC5RRAMAAAAA4BJFNAAAAAAALlFEAwAAAADgEt25G0D1js8dscQdgc2R93YAg/10FO5+n73T7OafP+566jUV9s6sHx05zRpvFFdpjQ9P2+6IHajKsI6N69PdGjeF9u7piG6m8qg13mz+Gmu80X/YcywQgxvZ59jy/z5wxJZ0D6yzPqJXzhx7B3nNsYc3+pnn46yzHLEXz/6hdezuS+zd6Rdf83trvGuSvSt2fdk0wf588bsr7Z1j/15kjx98PdcRy3nEz+8bYWPO72mNx73/j3o75u6Lk6zx7IQU13O8dcT+eqLr0/us8WrXMwPhMaqD/TWvz9LS2l93en/XTv2Ntx2zMFTXX6OsE7cNV6IBAAAAAHCJIhoAAAAAAJcoogEAAAAAcIkiGgAAAAAAlyiiAQAAAABwie7cscjYW95d/gN7d+NA/PKlm6zxvJn2ueMb2TvHvnzuJc6xqzf6OSpduCElnNnVGv9xkz9Zoo1DcszLM5y5t0QXhGRueEf1/v2OWOJyZ0yS8pbb55h+52D7HfHOb2M4d9231qG/zaq/jsq3tdgaUHzv6c41/uIR9lakqWpsf5lo758dmMS8Dtb4PZcvCHruh3YOt8aTLd+WAkSSsmvPs8Z/26rAGo+X8zngjJU3WMcmfppqjX/8y7nW+D2tNjpiw4f80jo2YcVH1rhfti8SirKO3VyJBgAAAADAJYpoAAAAAABcoogGAAAAAMAlimgAAAAAAFyisVgMqh7S2xqflfWUn0ckuJ476ZCtE4B/vvJya9x/EzHArnLuEWu8faL7JmKd//4ra3xd/mPWeFZCiiN25I0869j0q/5ljVeXlblcHeCfqTzqeqzPBPb/tD893h/riGX9j70xTdG19v/r0xtXWOOHP27uiJ2m4JtfIrSS3iqst7k//WWuNX55+r6g5/72z/a5k/VF0HMD9enMqZuscZ98fh7hvB7a6m/O1y6StO8ye9PJQOYOmShrImbDlWgAAAAAAFyiiAYAAAAAwCWKaAAAAAAAXKKIBgAAAADAJYpoAAAAAABcojt3DPIl2DuzJsW578LtT4dnd1jj1UHPDJzc53tb2u843RnaW23vQHnG7BJr/LKcn1vjq89e6Iit6PGydeyPWlxuXx/duRGl8iYWOWLV+/fbxy4ObO6suiwIMcV0tP8/DXjZM+3escZ9fq57dn19gjP20lrr2H2X9bLG4/3MXVzt/FaUxG8qrWMDbrZtK1WirGM3V6IBAAAAAHCJIhoAAAAAAJcoogEAAAAAcIkiGgAAAAAAlwIqomfPnq2+ffsqIyNDrVq10ujRo7V9+/ZaY8rLyzVp0iS1aNFC6enpGjNmjIqLi0O6aCBc2APwMvIfXkb+w+vYA8AJAXXnXrVqlSZNmqS+ffuqqqpKd9xxh4YNG6YtW7YoLS1NkjR9+nS99tprWrhwoTIzMzV58mRdccUVeu+99+rlBOBU3Dcl3EuIWeyB8GnR7BvXY3+08RfWeNYn263xoi/72ic62/UhPYH8jzyHrj7PERvX/Pd+RqfW72JiHPlfdwlZzn7sbw8s8DM6sNcwPvkcsfiqKGvzGyXYA/XP56dFtS3PJan94uA/VOxv7if+db4jZj74OOjjSVLK3kPOoLGfe6R+A1BARfSyZctq/Tx//ny1atVKhYWFGjRokEpLS/XHP/5RL774oi6++GJJ0rx583TGGWdo7dq1Ou8855M9EE3YA/Ay8h9eRv7D69gDwAlBvX1RWloqSWrevLkkqbCwUJWVlcrPz68Zc/rpp6t9+/Zas2aNdY6KigqVlZXVugHRgj0ALyP/4WWhyH+JPYDoxXMAvKzORbTP59O0adM0cOBA9ejRQ5JUVFSk5ORkNW3atNbY7OxsFRUVWeeZPXu2MjMza27t2rWr65KABsUegJeR//CyUOW/xB5AdOI5AF5X5yJ60qRJ2rRpkxYsWBDUAmbOnKnS0tKa2+7du4OaD2go7AF4GfkPLwtV/kvsAUQnngPgdQH9TfRxkydP1pIlS/TOO++obdu2NfGcnBwdPXpUJSUltd6FKi4uVk5OjnWulJQUpaTQCAvRhT0ALyP/4WWhzH+JPYDow3MAEGARbYzRlClTtGjRIq1cuVJ5eXm17u/Tp4+SkpK0fPlyjRkzRpK0fft2ffnllxowYEDoVo2TqurlvosxAsMeCJ/MRuWux77W64/W+BXX/toa//WFr9VpTV5D/keef13xrSPWKTGwLtx/PmR/cWvK3e85LyD/627rf5/miGUlhKZw+k1xP0esyYtrQzI3amMPhM7Xtzs7X0tSUtxGa7zST8P5xG+dvat3zO1vHfv54Kf9zG3/YPJfCp17q6s+tC8kQNVbPgnJPOEUUBE9adIkvfjii1q8eLEyMjJq/r4hMzNTqampyszM1C9+8QvNmDFDzZs3V5MmTTRlyhQNGDCAjnyICewBeBn5Dy8j/+F17AHghICK6CeffFKSNHjw4FrxefPmady4cZKkRx55RPHx8RozZowqKio0fPhwPfHEEyFZLBBu7AF4GfkPLyP/4XXsAeCEgD/OfSqNGjVSQUGBCgoK6rwoIFKxB+Bl5D+8jPyH17EHgBOC+p5oAAAAAAC8pE7duRE54hKd/4SjOm8Kw0qA+vXl2rb2O053hlolNLYOXf1Q8B8pKzzqbOIhSTpCEyZEp47J+6zxuIQzGngliFWX9t5Yb3NvvraTJfppvR0PCIXDnY9a45XG/hrDJ581vuz5Z1yP9ddAzN/49ou51noy/HYAAAAAAHCJIhoAAAAAAJcoogEAAAAAcIkiGgAAAAAAlyiiAQAAAABwie7cUc7Wnfv+nPdCMvf0Pec7YubQoZDMDQSq4/32rvMrr0lyxAY3qgzJMavk7JJ53V9vto7tWLQmJMcEbOJSUqzxpefZOs7bu9P7M3HjddZ4m5LNAc0D+PPZ5dmO2JylZ1rHTmu+xRo/56mp1ni77e/XfWFAmLRY43ztIklJIxOs8Uo/X9GdFOccH8hYSRr4z6us8fRX19sngiSuRAMAAAAA4BpFNAAAAAAALlFEAwAAAADgEkU0AAAAAAAuUUQDAAAAAOAS3bmjnK+iwhE7++kp1rHzxs21xhvFVVnjm+882xFLLv8ggNUBoePz0xl+yrwJjtjHNz0ekmMO/uc1jljH2+jCjYYXFxdnjZ+W6L4T9/Ij9g7fHWbY95b9mQEIXNXurxyxt89Ks459W32t8XaiCzdiR4s/2l9LdL54vDX+3IA/WuP9UpzfIuKTzzq24/9OtMZP/81Wa9w5M76LK9EAAAAAALhEEQ0AAAAAgEsU0QAAAAAAuEQRDQAAAACASxTRAAAAAAC4RHfuaGeMI9T+bnsHy1l39wlo6mTRiRuRr929znwfeW/vkMzdRJ+FZB4gWLZvYpCkHs9MdsSeG/eodexdn1xmjWfu+rTuCwMAhEyn6zZY43cr+Nc1XbTOGqcLd91wJRoAAAAAAJcoogEAAAAAcIkiGgAAAAAAlyiiAQAAAABwicZiAABEOksTSUlq/1tnY707f9vXOjZTNBADACAUuBINAAAAAIBLFNEAAAAAALhEEQ0AAAAAgEsU0QAAAAAAuEQRDQAAAACASxTRAAAAAAC4RBENAAAAAIBLFNEAAAAAALhEEQ0AAAAAgEsU0QAAAAAAuJQY7gV8nzFGklSlSsmEeTHwpCpVSjqRiw2NPYBwIv/hdewBeBn5Dy8LJP8jrog+dOiQJGm1loZ5JfC6Q4cOKTMzMyzHldgDCC/yH17HHoCXkf/wMjf5H2fC9VaTHz6fT3v27FFGRoYOHTqkdu3aaffu3WrSpEm4l1YvysrKYv4cpeg6T2OMDh06pNzcXMXHN/xfPLAHYk80nSP537CiKTeCEU3nGSl7wBij9u3bR8XvLBjRlBt1FU3nGCn5z3NA7Iimcwwk/yPuSnR8fLzatm0rSYqLi5MkNWnSJOJ/6cHywjlK0XOe4Xj39Tj2QOyKlnMk/xueF85Rip7zjIQ9UFZWJil6fmfB8sJ5Rss5RkL+SzwHxJpoOUe3+U9jMQAAAAAAXKKIBgAAAADApYguolNSUjRr1iylpKSEeyn1xgvnKHnnPEPNC783zhH+eOH35oVzlLxznqHkld+ZF87TC+dYH7zwe+Mco1fENRYDAAAAACBSRfSVaAAAAAAAIglFNAAAAAAALlFEAwAAAADgEkU0AAAAAAAuRXQRXVBQoNNOO02NGjVS//79tX79+nAvqc7eeecdXXrppcrNzVVcXJxeeeWVWvcbY3TXXXepdevWSk1NVX5+vnbs2BGexdbR7Nmz1bdvX2VkZKhVq1YaPXq0tm/fXmtMeXm5Jk2apBYtWig9PV1jxoxRcXFxmFYc2WIp/6XY3wPkf+jF0h6I9fyX2AOhRv6T/14WS/kvsQeOi6U9ELFF9EsvvaQZM2Zo1qxZ+uijj9SzZ08NHz5c+/btC/fS6uTw4cPq2bOnCgoKrPc/8MADeuyxx/TUU09p3bp1SktL0/Dhw1VeXt7AK627VatWadKkSVq7dq3efPNNVVZWatiwYTp8+HDNmOnTp+vVV1/VwoULtWrVKu3Zs0dXXHFFGFcdmWIt/6XY3wPkf2jF2h6I9fyX2AOhRP6T/14Wa/kvsQeOi6k9YCJUv379zKRJk2p+rq6uNrm5uWb27NlhXFVoSDKLFi2q+dnn85mcnBzz4IMP1sRKSkpMSkqK+ctf/hKGFYbGvn37jCSzatUqY8yxc0pKSjILFy6sGbN161YjyaxZsyZcy4xIsZz/xnhjD5D/wYnlPeCF/DeGPRAM8p/897JYzn9j2AOxsgci8kr00aNHVVhYqPz8/JpYfHy88vPztWbNmjCurH7s3LlTRUVFtc43MzNT/fv3j+rzLS0tlSQ1b95cklRYWKjKyspa53n66aerffv2UX2eoea1/Jdicw+Q/3XntT0Qi/kvsQfqivwn/73Ma/kvsQeidQ9EZBF94MABVVdXKzs7u1Y8OztbRUVFYVpV/Tl+TrF0vj6fT9OmTdPAgQPVo0cPScfOMzk5WU2bNq01NprPsz54Lf+l2NsD5H9wvLYHYi3/JfZAMMh/1fwcredL/ted1/JfYg9E63kmhnsBiE2TJk3Spk2btHr16nAvBWhw5D+8jj0ALyP/4XVe2AMReSW6ZcuWSkhIcHRrKy4uVk5OTphWVX+On1OsnO/kyZO1ZMkSrVixQm3btq2J5+Tk6OjRoyopKak1PlrPs754Lf+l2NoD5H/wvLYHYin/JfZAsMh/1fwcjedL/gfHa/kvsQei9TwjsohOTk5Wnz59tHz58pqYz+fT8uXLNWDAgDCurH7k5eUpJyen1vmWlZVp3bp1UXW+xhhNnjxZixYt0ttvv628vLxa9/fp00dJSUm1znP79u368ssvo+o865vX8l+KjT1A/oeO1/ZALOS/xB4IFfKf/Pcyr+W/xB6I2j0Q1rZmJ7FgwQKTkpJi5s+fb7Zs2WJuuOEG07RpU1NUVBTupdXJoUOHzIYNG8yGDRuMJPPwww+bDRs2mC+++MIYY8z9999vmjZtahYvXmz++c9/mssuu8zk5eWZI0eOhHnl7t10000mMzPTrFy50uzdu7fm9u2339aMufHGG0379u3N22+/bT788EMzYMAAM2DAgDCuOjLFWv4bE/t7gPwPrVjbA7Ge/8awB0KJ/Cf/vSzW8t8Y9sBxsbQHIraINsaYuXPnmvbt25vk5GTTr18/s3bt2nAvqc5WrFhhJDluY8eONcYca29/5513muzsbJOSkmKGDh1qtm/fHt5FB8h2fpLMvHnzasYcOXLETJw40TRr1sw0btzYXH755Wbv3r3hW3QEi6X8Nyb29wD5H3qxtAdiPf+NYQ+EGvlP/ntZLOW/MeyB42JpD8QZY0xormkDAAAAABDbIvJvogEAAAAAiEQU0QAAAAAAuEQRDQAAAACASxTRAAAAAAC4RBENAAAAAIBLFNEAAAAAALhEEQ0AAAAAgEsU0QAAAAAAuEQRDQAAAACASxTRAAAAAAC4RBENAAAAAIBLFNEAAAAAALj0/wFzSMvHqRecagAAAABJRU5ErkJggg==\n", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "fig, ax = plt.subplots(2,5)\n", - "for i, ax in enumerate(ax.flatten()):\n", - " img = np.array(preds[i].data).reshape(28,28)\n", - " prediction = np.argmax(preds[i].preds)\n", - " ax.set_title(f\"Prediction: {prediction}\")\n", - " ax.imshow(img)\n", - "\n", - "plt.rcParams[\"figure.figsize\"] = (12,7)\n", - "plt.show()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "883a1bc9-e091-4f11-869d-ef850cacf480", - "showTitle": false, - "title": "" - } - }, - "source": [ - "## Advanced\n", - "For most real-world use cases with Spark and DL, the training datasets will generally be much larger than can be hosted on a single box. Instead, they will typically be stored in a distributed filesystem, like HDFS, S3, or GCS. The following code demonstrates how to access a distributed dataset (hosted in S3) during training.\n", - "\n", - "**Note**: For simplicity, we will continue to use the MNIST dataset, but keep in mind that real-world datasets will be several orders of magnitude larger." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "ae4442ad-1a23-4732-9398-c5b580d85d29", - "showTitle": false, - "title": "" - } - }, - "source": [ - "### Convert Train Data to Parquet\n", - "\n", - "To emulate the use case where the dataset resides on a distributed file system, save the dataset as Parquet files to an S3 bucket.\n", - "\n", - "**Note**: this only needs to be run once, so you can skip this section if you've already generated the dataset on S3." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "445a0adf-ba40-4230-a923-cfd0cfe5730d", - "showTitle": false, - "title": "" - } - }, - "outputs": [], - "source": [ - "# save train dataset as parquet to S3\n", - "train_data_loader = get_train_dataloader(batch_size=60000, distributed=False)\n", - "save_as_parquet(train_data_loader, f\"{S3_BUCKET}/train\")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "ef83e46a-a4f9-44d1-9ca2-d9287aaf7f2e", - "showTitle": false, - "title": "" - } - }, - "source": [ - "### Distributed Data Loader\n", - "Since the dataset is stored in a distributed filesystem, you will need to use a distributed data loader capable of sharding files across the workers. Currently, there are two main options (aside from custom data loaders):\n", - "- Petastorm\n", - "- NVTabular\n", - "\n", - "Since the cluster is already configured with GPUs, we will demonstrate using NVTabular here, since it provides GPU-accelerated data loading with built-in support for S3FS and Parquet files." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "2220bcd9-8cb8-4671-a613-f5f2c1f985bf", - "showTitle": false, - "title": "" - } - }, - "outputs": [], - "source": [ - "storage_options={\"key\": access_key, \"secret\": secret_key}\n", - "\n", - "def seed_fn():\n", - " \"\"\"\n", - " Generate consistent dataloader shuffle seeds across workers\n", - " Reseeds each worker's dataloader each epoch to get fresh a shuffle\n", - " that's consistent across workers.\n", - " \"\"\"\n", - " return 0\n", - "\n", - "def get_train_dataloader(batch_size,):\n", - " import nvtabular as nvt\n", - " from nvtabular.loader.torch import TorchAsyncItr, DLDataLoader\n", - "\n", - " global_rank = int(os.environ[\"RANK\"])\n", - " world_size = int(os.environ[\"WORLD_SIZE\"])\n", - "\n", - " train_dataset = TorchAsyncItr(\n", - " nvt.Dataset(f\"{S3_BUCKET}/train\", engine=\"parquet\", storage_options=storage_options),\n", - " batch_size=batch_size,\n", - " cats=[],\n", - " conts=[\"data\"],\n", - " labels=[\"label\"],\n", - " shuffle=True,\n", - " parts_per_chunk=1,\n", - " global_size=world_size,\n", - " global_rank=global_rank,\n", - " seed_fn=seed_fn\n", - " )\n", - " train_loader = DLDataLoader(\n", - " train_dataset, batch_size=None, collate_fn=lambda x: x, pin_memory=False, num_workers=0\n", - " )\n", - "\n", - " return train_loader\n", - "\n", - "\n", - "def get_test_dataloader(batch_size):\n", - " import nvtabular as nvt\n", - " from nvtabular.loader.torch import TorchAsyncItr, DLDataLoader\n", - "\n", - " global_rank = int(os.environ[\"RANK\"])\n", - " world_size = int(os.environ[\"WORLD_SIZE\"])\n", - "\n", - " train_dataset = TorchAsyncItr(\n", - " nvt.Dataset(f\"{S3_BUCKET}/test\", engine=\"parquet\", storage_options=storage_options),\n", - " batch_size=batch_size,\n", - " cats=[],\n", - " conts=[\"data\"],\n", - " labels=[\"label\"],\n", - " shuffle=False,\n", - " parts_per_chunk=1,\n", - " global_size=world_size,\n", - " global_rank=global_rank,\n", - " seed_fn=seed_fn\n", - " )\n", - " train_loader = DLDataLoader(\n", - " train_dataset, batch_size=None, collate_fn=lambda x: x, pin_memory=False, num_workers=0\n", - " )\n", - "\n", - " return train_loader\n", - "\n", - "def prepare_batch(batch):\n", - " data = batch[0]['data__values'].reshape(-1,1,28,28)\n", - " label = batch[1]\n", - " return data, label" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "4b98395c-a662-4d4f-89a2-7837af89a9ec", - "showTitle": false, - "title": "" - } - }, - "source": [ - "### Train with NVTabular Data Loader" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "2ebb1fb7-f7cc-4601-8e8b-9fb26f4072dc", - "showTitle": false, - "title": "" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Log directory: /dbfs/ml/pytorch/1683755103.642704\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Started distributed training with 2 executor proceses\n", - "Running distributed training\n", - "Running distributed training\n", - "adjusted_batch_size=50\n", - "adjusted_batch_size=50\n", - "2023-05-10 21:45:14.407035: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F AVX512_VNNI FMA\n", - "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "2023-05-10 21:45:14.551173: I tensorflow/core/util/port.cc:104] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n", - "2023-05-10 21:45:15.573393: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F AVX512_VNNI FMA\n", - "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "2023-05-10 21:45:15.733816: I tensorflow/core/util/port.cc:104] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n", - "/databricks/python/lib/python3.10/site-packages/tritonclient/grpc/__init__.py:60: UserWarning: Imported version of grpc is 1.48.1. There is a memory leak in certain Python GRPC versions (1.43.0 to be specific). Please use versions <1.43.0 or >=1.51.1 to avoid leaks (see https://github.com/grpc/grpc/issues/28513).\n", - " warnings.warn(\n", - "2023-05-10 21:45:16.309734: I tensorflow/compiler/xla/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n", - "2023-05-10 21:45:16.310694: I tensorflow/compiler/xla/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n", - "2023-05-10 21:45:16.311293: I tensorflow/compiler/xla/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n", - "/databricks/python/lib/python3.10/site-packages/tritonclient/grpc/__init__.py:60: UserWarning: Imported version of grpc is 1.48.1. There is a memory leak in certain Python GRPC versions (1.43.0 to be specific). Please use versions <1.43.0 or >=1.51.1 to avoid leaks (see https://github.com/grpc/grpc/issues/28513).\n", - " warnings.warn(\n", - "2023-05-10 21:45:17.727073: I tensorflow/compiler/xla/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n", - "2023-05-10 21:45:17.728005: I tensorflow/compiler/xla/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n", - "2023-05-10 21:45:17.728554: I tensorflow/compiler/xla/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n", - "Train Epoch: 1 [0]\tLoss: 2.305805\n", - "Train Epoch: 1 [0]\tLoss: 2.339357\n", - "Train Epoch: 1 [5000]\tLoss: 2.280867\n", - "Train Epoch: 1 [5000]\tLoss: 2.368357\n", - "Train Epoch: 1 [10000]\tLoss: 2.290627\n", - "Train Epoch: 1 [10000]\tLoss: 2.271626\n", - "Train Epoch: 1 [15000]\tLoss: 2.285692\n", - "Train Epoch: 1 [15000]\tLoss: 2.309556\n", - "Train Epoch: 1 [20000]\tLoss: 2.291195\n", - "Train Epoch: 1 [20000]\tLoss: 2.258479\n", - "Train Epoch: 1 [25000]\tLoss: 2.305413\n", - "Train Epoch: 1 [25000]\tLoss: 2.257723\n", - "Train Epoch: 2 [0]\tLoss: 2.235390\n", - "Train Epoch: 2 [0]\tLoss: 2.198956\n", - "Train Epoch: 2 [5000]\tLoss: 2.218740\n", - "Train Epoch: 2 [5000]\tLoss: 2.221195\n", - "Train Epoch: 2 [10000]\tLoss: 2.187166\n", - "Train Epoch: 2 [10000]\tLoss: 2.137402\n", - "Train Epoch: 2 [15000]\tLoss: 2.126111\n", - "Train Epoch: 2 [15000]\tLoss: 2.073809\n", - "Train Epoch: 2 [20000]\tLoss: 2.081784\n", - "Train Epoch: 2 [20000]\tLoss: 1.966479\n", - "Train Epoch: 2 [25000]\tLoss: 1.930605\n", - "Train Epoch: 2 [25000]\tLoss: 1.878074\n", - "Train Epoch: 3 [0]\tLoss: 1.694411\n", - "Train Epoch: 3 [0]\tLoss: 1.862214\n", - "Train Epoch: 3 [5000]\tLoss: 1.893949\n", - "Train Epoch: 3 [5000]\tLoss: 1.882721\n", - "Train Epoch: 3 [10000]\tLoss: 1.315645\n", - "Train Epoch: 3 [10000]\tLoss: 1.813671\n", - "Train Epoch: 3 [15000]\tLoss: 1.548722\n", - "Train Epoch: 3 [15000]\tLoss: 1.306659\n", - "Train Epoch: 3 [20000]\tLoss: 1.352624\n", - "Train Epoch: 3 [20000]\tLoss: 1.249572\n", - "Train Epoch: 3 [25000]\tLoss: 1.214441\n", - "Train Epoch: 3 [25000]\tLoss: 1.272143\n", - "Train Epoch: 4 [0]\tLoss: 1.395404\n", - "Train Epoch: 4 [0]\tLoss: 1.208001\n", - "Train Epoch: 4 [5000]\tLoss: 1.393423\n", - "Train Epoch: 4 [5000]\tLoss: 1.033216\n", - "Train Epoch: 4 [10000]\tLoss: 1.039255\n", - "Train Epoch: 4 [10000]\tLoss: 1.147518\n", - "Train Epoch: 4 [15000]\tLoss: 1.105749\n", - "Train Epoch: 4 [15000]\tLoss: 0.898317\n", - "Train Epoch: 4 [20000]\tLoss: 0.889672\n", - "Train Epoch: 4 [20000]\tLoss: 0.999275\n", - "Train Epoch: 4 [25000]\tLoss: 0.814816\n", - "Train Epoch: 4 [25000]\tLoss: 0.955990\n", - "Train Epoch: 5 [0]\tLoss: 1.236833\n", - "Train Epoch: 5 [0]\tLoss: 0.852844\n", - "Train Epoch: 5 [5000]\tLoss: 1.068169\n", - "Train Epoch: 5 [5000]\tLoss: 0.947117\n", - "Train Epoch: 5 [10000]\tLoss: 1.434625\n", - "Train Epoch: 5 [10000]\tLoss: 0.652500\n", - "Train Epoch: 5 [15000]\tLoss: 0.938174\n", - "Train Epoch: 5 [15000]\tLoss: 0.902604\n", - "Train Epoch: 5 [20000]\tLoss: 0.698800\n", - "Train Epoch: 5 [20000]\tLoss: 0.812113\n", - "Train Epoch: 5 [25000]\tLoss: 0.591616\n", - "Train Epoch: 5 [25000]\tLoss: 1.005589\n", - "Average test loss: 0.5255906581878662\n", - "Finished distributed training with 2 executor proceses\n" - ] - } - ], - "source": [ - "from pyspark.ml.torch.distributor import TorchDistributor\n", - "\n", - "multi_node_multi_gpu_dir = create_log_dir()\n", - "print(\"Log directory: \", create_log_dir())\n", - "\n", - "# Set num_processes = NUM_WORKERS * NUM_GPUS_PER_WORKER\n", - "output_dist = TorchDistributor(num_processes=2, local_mode=False, use_gpu=True).run(main_fn, multi_node_multi_gpu_dir)\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "7e66fc07-ee69-4cdc-81d6-3130e03df3b2", - "showTitle": false, - "title": "" - } - }, - "outputs": [], - "source": [] - } - ], - "metadata": { - "application/vnd.databricks.v1+notebook": { - "dashboards": [], - "language": "python", - "notebookMetadata": { - "mostRecentlyExecutedCommandWithImplicitDF": { - "commandId": 2919187681861530, - "dataframes": [ - "_sqldf" - ] - }, - "pythonIndentUnit": 2 - }, - "notebookName": "Spark 3.4 E2E S3", - "notebookOrigID": 2285149241771137, - "widgets": {} - }, - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.16" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/examples/ML+DL-Examples/Spark-cuML/pca/Dockerfile b/examples/ML+DL-Examples/Spark-cuML/pca/Dockerfile index 12d00546c..deaef0ffd 100644 --- a/examples/ML+DL-Examples/Spark-cuML/pca/Dockerfile +++ b/examples/ML+DL-Examples/Spark-cuML/pca/Dockerfile @@ -18,7 +18,7 @@ ARG CUDA_VER=11.8.0 FROM nvidia/cuda:${CUDA_VER}-devel-ubuntu20.04 # Please do not update the BRANCH_VER version -ARG BRANCH_VER=24.06 +ARG BRANCH_VER=24.08 RUN apt-get update RUN apt-get install -y wget ninja-build git diff --git a/examples/SQL+DF-Examples/micro-benchmarks/README.md b/examples/SQL+DF-Examples/micro-benchmarks/README.md index 29a17061e..e135ff239 100644 --- a/examples/SQL+DF-Examples/micro-benchmarks/README.md +++ b/examples/SQL+DF-Examples/micro-benchmarks/README.md @@ -3,7 +3,7 @@ Standard industry benchmarks are a great way to measure performance over a period of time but another barometer to measure performance is to measure performance of common operators that are used in the data preprocessing stage or in data analytics. -The microbenchmark notebook in this repo uses four such queries in the chart shown below: +The microbenchmark notebook in this repo uses five such queries in the chart shown below: - **Count Distinct**: a function used to estimate the number of unique page views or unique customers visiting an e-commerce site. @@ -11,6 +11,7 @@ The microbenchmark notebook in this repo uses four such queries in the chart sho timestamped event data in marketing or financial industry. - **Intersect**: an operator used to remove duplicates in a dataframe. - **Cross-join**: A common use for a cross join is to obtain all combinations of items. +- **Hash-join**: Joining two tables together by matching rows based on a common column. These queries were run on a standard eight-nodes CPU cluster with 2 CPU (128 cores), 512GB memory and 1xA100 GPUs per node. The dataset used was of size 3TB with multiple different data types. diff --git a/examples/SQL+DF-Examples/micro-benchmarks/notebooks/micro-benchmarks-cpu.ipynb b/examples/SQL+DF-Examples/micro-benchmarks/notebooks/micro-benchmarks-cpu.ipynb index ce5c5a797..ae53e6f5d 100644 --- a/examples/SQL+DF-Examples/micro-benchmarks/notebooks/micro-benchmarks-cpu.ipynb +++ b/examples/SQL+DF-Examples/micro-benchmarks/notebooks/micro-benchmarks-cpu.ipynb @@ -529,6 +529,34 @@ "runMicroBenchmark(spark,\"Crossjoin\",query ,1)" ] }, + { + "cell_type": "markdown", + "id": "56f915c2-9b9a-4982-8c4e-5b570c17bfeb", + "metadata": {}, + "source": [ + "### HashJoin\n", + "This is a microbenchmark for a HashJoin. The query on GPU will be more than 10x times faster than CPU based on the cluster in the readme." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "040603c9-a96f-4017-bcdb-5f93e12996a4", + "metadata": {}, + "outputs": [], + "source": [ + "spark.read.parquet(dataRoot + \"/tpcds/store_sales\").createOrReplaceTempView(\"store_sales\")\n", + "spark.read.parquet(dataRoot + \"/tpcds/store_returns\").createOrReplaceTempView(\"store_returns\")\n", + "\n", + "print(\"-\"*50)\n", + "query = '''\n", + "select sum(store_sales.ss_ext_wholesale_cost)\n", + "from store_sales\n", + "join store_returns on (ss_item_sk = sr_item_sk) and (ss_addr_sk=sr_addr_sk)\n", + "'''\n", + "runMicroBenchmark(spark,\"HashJoin\",query,1)" + ] + }, { "cell_type": "code", "execution_count": 24, @@ -550,7 +578,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -564,7 +592,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.2" + "version": "3.12.3" } }, "nbformat": 4, diff --git a/examples/SQL+DF-Examples/micro-benchmarks/notebooks/micro-benchmarks-gpu.ipynb b/examples/SQL+DF-Examples/micro-benchmarks/notebooks/micro-benchmarks-gpu.ipynb index 099b047a4..e2c2b4a6f 100644 --- a/examples/SQL+DF-Examples/micro-benchmarks/notebooks/micro-benchmarks-gpu.ipynb +++ b/examples/SQL+DF-Examples/micro-benchmarks/notebooks/micro-benchmarks-gpu.ipynb @@ -22,7 +22,7 @@ "import os\n", "# Change to your cluster ip:port and directories\n", "SPARK_MASTER_URL = os.getenv(\"SPARK_MASTER_URL\", \"spark:your-ip:port\")\n", - "RAPIDS_JAR = os.getenv(\"RAPIDS_JAR\", \"/your-path/rapids-4-spark_2.12-24.06.0.jar\")\n" + "RAPIDS_JAR = os.getenv(\"RAPIDS_JAR\", \"/your-path/rapids-4-spark_2.12-24.08.1.jar\")\n" ] }, { @@ -541,18 +541,48 @@ "runMicroBenchmark(spark,\"Crossjoin\",query,2)" ] }, + { + "cell_type": "markdown", + "id": "06b351e6-b7bd-4063-a20b-fe4fd71221f9", + "metadata": {}, + "source": [ + "### HashJoin\n", + "This is a microbenchmark for a HashJoin. The query on GPU will be more than 10x times faster than CPU based on the cluster in the readme." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "191d0c9a-2d3a-40f4-89aa-f61dab5caa90", + "metadata": {}, + "outputs": [], + "source": [ + "spark.read.parquet(dataRoot + \"/tpcds/store_sales\").createOrReplaceTempView(\"store_sales\")\n", + "spark.read.parquet(dataRoot + \"/tpcds/store_returns\").createOrReplaceTempView(\"store_returns\")\n", + "\n", + "print(\"-\"*50)\n", + "query = '''\n", + "select sum(store_sales.ss_ext_wholesale_cost)\n", + "from store_sales\n", + "join store_returns on (ss_item_sk = sr_item_sk) and (ss_addr_sk=sr_addr_sk)\n", + "'''\n", + "runMicroBenchmark(spark,\"HashJoin\",query,1)" + ] + }, { "cell_type": "code", "execution_count": null, "id": "fc2092e8", "metadata": {}, "outputs": [], - "source": [] + "source": [ + "spark.stop()" + ] } ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -566,7 +596,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.2" + "version": "3.12.3" } }, "nbformat": 4, diff --git a/examples/UDF-Examples/RAPIDS-accelerated-UDFs/Dockerfile b/examples/UDF-Examples/RAPIDS-accelerated-UDFs/Dockerfile index b5ef1cc0c..f1c252fb5 100644 --- a/examples/UDF-Examples/RAPIDS-accelerated-UDFs/Dockerfile +++ b/examples/UDF-Examples/RAPIDS-accelerated-UDFs/Dockerfile @@ -16,72 +16,54 @@ # A container that can be used to build UDF native code against libcudf ARG CUDA_VERSION=11.8.0 -ARG LINUX_VERSION=ubuntu18.04 +ARG LINUX_VERSION=rockylinux8 FROM nvidia/cuda:${CUDA_VERSION}-devel-${LINUX_VERSION} -ARG DEBIAN_FRONTEND=noninteractive +ARG TOOLSET_VERSION=11 +ENV TOOLSET_VERSION=11 ARG PARALLEL_LEVEL=10 ENV PARALLEL_LEVEL=10 -RUN GCC_VERSION=$(bash -c '\ -CUDA_VERSION=$(nvcc --version | head -n4 | tail -n1 | cut -d" " -f5 | cut -d"," -f1); \ -CUDA_VERSION_MAJOR=$(echo $CUDA_VERSION | tr -d '.' | cut -c 1-2); \ -CUDA_VERSION_MINOR=$(echo $CUDA_VERSION | tr -d '.' | cut -c 3); \ - if [[ "$CUDA_VERSION_MAJOR" == 9 ]]; then echo "7"; \ - elif [[ "$CUDA_VERSION_MAJOR" == 10 ]]; then echo "8"; \ - elif [[ "$CUDA_VERSION_MAJOR" == 11 ]]; then echo "9"; \ - else echo "10"; \ - fi') \ -&& apt update -y \ -&& apt install -y software-properties-common \ -&& add-apt-repository -y ppa:git-core/ppa \ -&& add-apt-repository -y ppa:ubuntu-toolchain-r/test \ -&& add-apt-repository ppa:deadsnakes/ppa \ -&& apt update -y \ -&& apt install -y \ - build-essential git rsync wget \ - gcc-${GCC_VERSION} g++-${GCC_VERSION} \ - openjdk-8-jdk maven tzdata \ - # CMake dependencies - curl libssl-dev libcurl4-openssl-dev zlib1g-dev \ -&& apt autoremove -y \ -&& rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \ -&& update-alternatives \ - --install /usr/bin/gcc gcc /usr/bin/gcc-${GCC_VERSION} 100 \ -&& update-alternatives \ - --install /usr/bin/g++ g++ /usr/bin/g++-${GCC_VERSION} 100 \ -# Set gcc-${GCC_VERSION} as the default gcc -&& update-alternatives --set gcc /usr/bin/gcc-${GCC_VERSION} \ -# Set gcc-${GCC_VERSION} as the default g++ -&& update-alternatives --set g++ /usr/bin/g++-${GCC_VERSION} \ -# Set JDK8 as the default Java -&& update-alternatives --set java /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java +### Install basic requirements +RUN dnf --enablerepo=powertools install -y \ + gcc-toolset-${TOOLSET_VERSION} \ + git \ + java-1.8.0-openjdk \ + maven \ + ninja-build \ + patch \ + python39 \ + scl-utils \ + tar \ + wget \ + zlib-devel \ + && alternatives --set python /usr/bin/python3 +# 3.22.3: CUDA architecture 'native' support + flexible CMAKE__*_LAUNCHER for ccache ARG CMAKE_VERSION=3.26.4 +# default x86_64 from x86 build, aarch64 cmake for arm build +ARG CMAKE_ARCH=x86_64 +RUN cd /usr/local && wget --quiet https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-${CMAKE_ARCH}.tar.gz && \ + tar zxf cmake-${CMAKE_VERSION}-linux-${CMAKE_ARCH}.tar.gz && \ + rm cmake-${CMAKE_VERSION}-linux-${CMAKE_ARCH}.tar.gz +ENV PATH /usr/local/cmake-${CMAKE_VERSION}-linux-${CMAKE_ARCH}/bin:$PATH -# Install CMake -RUN cd /tmp \ - && curl -fsSLO --compressed "https://github.com/Kitware/CMake/releases/download/v$CMAKE_VERSION/cmake-$CMAKE_VERSION.tar.gz" -o /tmp/cmake-$CMAKE_VERSION.tar.gz \ - && tar -xvzf /tmp/cmake-$CMAKE_VERSION.tar.gz && cd /tmp/cmake-$CMAKE_VERSION \ - && /tmp/cmake-$CMAKE_VERSION/bootstrap \ - --system-curl \ - --parallel=${PARALLEL_LEVEL} \ - && make install -j${PARALLEL_LEVEL} \ - && cd /tmp && rm -rf /tmp/cmake-$CMAKE_VERSION* - -# Install ccache +# ccache for interactive builds ARG CCACHE_VERSION=4.6 RUN cd /tmp && wget --quiet https://github.com/ccache/ccache/releases/download/v${CCACHE_VERSION}/ccache-${CCACHE_VERSION}.tar.gz && \ - tar zxf ccache-${CCACHE_VERSION}.tar.gz && \ - rm ccache-${CCACHE_VERSION}.tar.gz && \ - cd ccache-${CCACHE_VERSION} && \ - mkdir build && \ - cd build && \ - cmake .. \ - -DCMAKE_BUILD_TYPE=Release \ - -DZSTD_FROM_INTERNET=ON \ - -DREDIS_STORAGE_BACKEND=OFF && \ - cmake --build . --parallel ${PARALLEL_LEVEL} --target install && \ - cd ../.. && \ - rm -rf ccache-${CCACHE_VERSION} + tar zxf ccache-${CCACHE_VERSION}.tar.gz && \ + rm ccache-${CCACHE_VERSION}.tar.gz && \ + cd ccache-${CCACHE_VERSION} && \ + mkdir build && \ + cd build && \ + scl enable gcc-toolset-${TOOLSET_VERSION} \ + "cmake .. \ + -DCMAKE_BUILD_TYPE=Release \ + -DZSTD_FROM_INTERNET=ON \ + -DREDIS_STORAGE_BACKEND=OFF && \ + cmake --build . --parallel ${PARALLEL_LEVEL} --target install" && \ + cd ../.. && \ + rm -rf ccache-${CCACHE_VERSION} + +ENTRYPOINT /usr/bin/scl enable gcc-toolset-${TOOLSET_VERSION} -- bash diff --git a/examples/UDF-Examples/RAPIDS-accelerated-UDFs/README.md b/examples/UDF-Examples/RAPIDS-accelerated-UDFs/README.md index 751057ed6..2a9fcf6cf 100644 --- a/examples/UDF-Examples/RAPIDS-accelerated-UDFs/README.md +++ b/examples/UDF-Examples/RAPIDS-accelerated-UDFs/README.md @@ -122,8 +122,8 @@ Run the following commands to build and start a docker ```bash cd spark-rapids-examples/examples/UDF-Examples/RAPIDS-accelerated-UDFs -docker build -t my-local:my-udf-example-ubuntu . -nvidia-docker run -it my-local:my-udf-example-ubuntu +docker build -t my-local:my-udf-example . +nvidia-docker run -it my-local:my-udf-example ``` ### Build the udf-examples jar @@ -139,7 +139,7 @@ export CCACHE_DIR="$LOCAL_CCACHE_DIR" export CMAKE_C_COMPILER_LAUNCHER="ccache" export CMAKE_CXX_COMPILER_LAUNCHER="ccache" export CMAKE_CUDA_COMPILER_LAUNCHER="ccache" -export CMAKE_CXX_LINKER_LAUNCHER="ccache +export CMAKE_CXX_LINKER_LAUNCHER="ccache" mvn clean package -Pudf-native-examples ``` @@ -186,7 +186,7 @@ then do the following inside the Docker container. ### Get jars from Maven Central -[rapids-4-spark_2.12-24.06.0.jar](https://repo1.maven.org/maven2/com/nvidia/rapids-4-spark_2.12/24.06.0/rapids-4-spark_2.12-24.06.0.jar) +[rapids-4-spark_2.12-24.08.1.jar](https://repo1.maven.org/maven2/com/nvidia/rapids-4-spark_2.12/24.08.1/rapids-4-spark_2.12-24.08.1.jar) ### Launch a local mode Spark @@ -206,9 +206,9 @@ $SPARK_HOME/bin/pyspark --master local[*] \ ### Test native based UDF -Input the following commands to test wordcount JIN UDF +Input the following commands to test wordcount JNI UDF -```bash +```python from pyspark.sql.types import * schema = StructType([ StructField("c1", StringType()), diff --git a/examples/UDF-Examples/RAPIDS-accelerated-UDFs/pom.xml b/examples/UDF-Examples/RAPIDS-accelerated-UDFs/pom.xml index 34d7e627b..6041cd8ce 100644 --- a/examples/UDF-Examples/RAPIDS-accelerated-UDFs/pom.xml +++ b/examples/UDF-Examples/RAPIDS-accelerated-UDFs/pom.xml @@ -25,7 +25,7 @@ user defined functions for use with the RAPIDS Accelerator for Apache Spark - 24.06.0-SNAPSHOT + 24.08.1-SNAPSHOT 1.8 @@ -37,7 +37,7 @@ cuda11 2.12 - 24.06.0 + 24.08.1 3.1.1 2.12.15 ${project.build.directory}/cpp-build diff --git a/examples/UDF-Examples/RAPIDS-accelerated-UDFs/src/main/cpp/CMakeLists.txt b/examples/UDF-Examples/RAPIDS-accelerated-UDFs/src/main/cpp/CMakeLists.txt index 4bdec9b00..af50f5727 100755 --- a/examples/UDF-Examples/RAPIDS-accelerated-UDFs/src/main/cpp/CMakeLists.txt +++ b/examples/UDF-Examples/RAPIDS-accelerated-UDFs/src/main/cpp/CMakeLists.txt @@ -16,7 +16,7 @@ cmake_minimum_required(VERSION 3.23.1 FATAL_ERROR) -file(DOWNLOAD https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-24.06/RAPIDS.cmake +file(DOWNLOAD https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-24.08/RAPIDS.cmake ${CMAKE_BINARY_DIR}/RAPIDS.cmake) include(${CMAKE_BINARY_DIR}/RAPIDS.cmake) @@ -32,7 +32,7 @@ if(DEFINED GPU_ARCHS) endif() rapids_cuda_init_architectures(UDFEXAMPLESJNI) -project(UDFEXAMPLESJNI VERSION 24.06.0 LANGUAGES C CXX CUDA) +project(UDFEXAMPLESJNI VERSION 24.08.1 LANGUAGES C CXX CUDA) option(PER_THREAD_DEFAULT_STREAM "Build with per-thread default stream" OFF) option(BUILD_UDF_BENCHMARKS "Build the benchmarks" OFF) @@ -84,10 +84,10 @@ set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -w --expt-extended-lambda --expt-relax set(CUDA_USE_STATIC_CUDA_RUNTIME ON) rapids_cpm_init() -rapids_cpm_find(cudf 24.06.00 +rapids_cpm_find(cudf 24.08.10 CPM_ARGS GIT_REPOSITORY https://github.com/rapidsai/cudf.git - GIT_TAG branch-24.06 + GIT_TAG branch-24.08 GIT_SHALLOW TRUE SOURCE_SUBDIR cpp OPTIONS "BUILD_TESTS OFF" diff --git a/examples/UDF-Examples/Spark-cuSpatial/.gitignore b/examples/UDF-Examples/Spark-cuSpatial/.gitignore deleted file mode 100644 index f5897467a..000000000 --- a/examples/UDF-Examples/Spark-cuSpatial/.gitignore +++ /dev/null @@ -1,65 +0,0 @@ -## Common -__pycache__ -*.py[cod] -*$py.class -*.a -*.o -*.so -*.dylib -.cache -.vscode -*.swp -*.pytest_cache -DartConfiguration.tcl -.DS_Store -*.manifest -*.spec -.nfs* -.clangd - -## Patching -*.diff -*.orig -*.rej - -## C++ build directories & artifacts -CMakeFiles/ -Debug -build/ - -## Eclipse IDE -.project -.cproject -.settings - -## IntelliJ IDE -.idea/ -.idea_modules/ -*.iml -*.ipr -*.iws - -#Java -target - -# Jupyter Notebook -.ipynb_checkpoints - -# pyenv -.python-version - -# dotenv -.env - -# virtualenv -.venv -venv/ -ENV/ - -## VSCode IDE -.vscode - -# protobuf -**/*_pb2.py - -cufile.log diff --git a/examples/UDF-Examples/Spark-cuSpatial/Dockerfile b/examples/UDF-Examples/Spark-cuSpatial/Dockerfile deleted file mode 100644 index 6d1afc4e6..000000000 --- a/examples/UDF-Examples/Spark-cuSpatial/Dockerfile +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/bash -# -# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -ARG CUDA_VER=11.2.2 -FROM nvidia/cuda:${CUDA_VER}-devel-ubuntu18.04 - -RUN apt-key adv --fetch-keys https://developer.download.nvidia.cn/compute/cuda/repos/ubuntu1804/x86_64/3bf863cc.pub -RUN apt-get update -RUN apt-get install -y wget ninja-build git - -ENV PATH="/root/miniconda3/bin:${PATH}" -ARG PATH="/root/miniconda3/bin:${PATH}" -RUN wget --quiet \ - https://repo.anaconda.com/miniconda/Miniconda3-py38_4.10.3-Linux-x86_64.sh \ - && mkdir /root/.conda \ - && bash Miniconda3-py38_4.10.3-Linux-x86_64.sh -b \ - && rm -f Miniconda3-py38_4.10.3-Linux-x86_64.sh \ - && conda init - -SHELL ["conda", "run", "--no-capture-output", "-n", "base", "/bin/bash", "-c"] -RUN echo $PATH -RUN echo $CONDA_PREFIX -RUN conda --version - -RUN conda install -c conda-forge openjdk=8 maven=3.8.1 -y - -RUN wget --quiet \ - https://github.com/Kitware/CMake/releases/download/v3.26.4/cmake-3.26.4-linux-x86_64.tar.gz \ - && tar -xzf cmake-3.21.3-linux-x86_64.tar.gz \ - && rm -rf cmake-3.21.3-linux-x86_64.tar.gz - -ENV PATH="/cmake-3.21.3-linux-x86_64/bin:${PATH}" \ No newline at end of file diff --git a/examples/UDF-Examples/Spark-cuSpatial/Dockerfile.awsdb b/examples/UDF-Examples/Spark-cuSpatial/Dockerfile.awsdb deleted file mode 100644 index 8f7bdc660..000000000 --- a/examples/UDF-Examples/Spark-cuSpatial/Dockerfile.awsdb +++ /dev/null @@ -1,67 +0,0 @@ -# -# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -FROM nvidia/cuda:11.2.2-devel-ubuntu18.04 - -ENV DEBIAN_FRONTEND=noninteractive - -# update cuda pub key to avoid GPG error -RUN apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/3bf863cc.pub - -# See https://github.com/databricks/containers/blob/master/ubuntu/minimal/Dockerfile -RUN apt-get update && \ - apt-get install --yes --no-install-recommends \ - openjdk-8-jdk \ - openjdk-8-jre \ - lsb-release \ - iproute2 \ - bash \ - sudo \ - coreutils \ - procps \ - wget && \ - /var/lib/dpkg/info/ca-certificates-java.postinst configure && \ - rm -rf /var/lib/apt/lists/* - -ENV PATH /databricks/conda/bin:$PATH - -RUN wget -q https://repo.continuum.io/miniconda/Miniconda3-py38_4.9.2-Linux-x86_64.sh -O miniconda.sh && \ - bash miniconda.sh -b -p /databricks/conda && \ - rm miniconda.sh && \ - # Source conda.sh for all login and interactive shells. - ln -s /databricks/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \ - echo ". /etc/profile.d/conda.sh" >> ~/.bashrc && \ - # Set always_yes for non-interactive shells. - conda config --system --set always_yes True && \ - conda clean --all - -RUN conda install -c rapidsai-nightly -c nvidia -c conda-forge -c defaults libcuspatial=23.02 -RUN conda install -c conda-forge libgdal==3.3.1 -RUN pip install jupyter -ENV JAVA_HOME /usr/lib/jvm/java-1.8.0-openjdk-amd64 -ENV PATH $PATH:/usr/lib/jvm/java-1.8.0-openjdk-amd64/jre/bin:/usr/lib/jvm/java-1.8.0-openjdk-amd64/bin - -RUN apt-get update && apt-get install wget openssh-client openssh-server \ - -y --allow-downgrades --allow-change-held-packages --no-install-recommends -RUN useradd --create-home --shell /bin/bash --groups sudo ubuntu - -ENV PYSPARK_PYTHON=/databricks/conda/bin/python -ENV USER root -ENV DEFAULT_DATABRICKS_ROOT_CONDA_ENV=base -ENV DATABRICKS_ROOT_CONDA_ENV=base -ENV LD_LIBRARY_PATH=/databricks/conda/lib:$LD_LIBRARY_PATH -# required by DB -RUN pip install virtualenv diff --git a/examples/UDF-Examples/Spark-cuSpatial/README.md b/examples/UDF-Examples/Spark-cuSpatial/README.md deleted file mode 100644 index 6ba27ae2e..000000000 --- a/examples/UDF-Examples/Spark-cuSpatial/README.md +++ /dev/null @@ -1,164 +0,0 @@ -# Spark-cuSpatial - -This is a Spark RapidsUDF application to illustrate how to use [cuSpatial](https://github.com/rapidsai/cuspatial) to solve a point-in-polygon problem. -It implements a [RapidsUDF](https://nvidia.github.io/spark-rapids/docs/additional-functionality/rapids-udfs.html#adding-gpu-implementations-to-udfs) -interface to call the cuSpatial functions through JNI. It can be run on a distributed Spark cluster with scalability. - -## Performance -We got the end-2-end hot run times as below table when running with 2009 NYC Taxi trip pickup location, -which includes 170,896,055 points, and 3 sets of polygons(taxi_zone, nyct2000, nycd Community-Districts). -The point data can be downloaded from [TLC Trip Record Data](https://www1.nyc.gov/site/tlc/about/tlc-trip-record-data.page). -The polygon data can be downloaded from [taxi_zone dataset](https://data.cityofnewyork.us/Transportation/NYC-Taxi-Zones/d3c5-ddgc), -[nyct2000 dataset](https://data.cityofnewyork.us/City-Government/2000-Census-Tracts/ysjj-vb9j) and -[nycd Community-Districts dataset](https://data.cityofnewyork.us/City-Government/Community-Districts/yfnk-k7r4) - -| Environment | Taxi_zones (263 Polygons) | Nyct2000 (2216 Polygons) | Nycd Community-Districts (71 Complex Polygons)| -| ----------- | :---------: | :---------: | :---------: | -| 4-core CPU | 3.9 minutes | 4.0 minutes| 4.1 minutes | -| 1 GPU(T4) on Databricks | 25 seconds | 27 seconds | 28 seconds| -| 2 GPU(T4) on Databricks | 15 seconds | 14 seconds | 17 seconds | -| 4 GPU(T4) on Databricks | 11 seconds | 11 seconds | 12 seconds | - -Note: Please update the `x,y` column names to `Start_Lon,Start_Lat` in -the [notebook](./notebooks/cuspatial_sample_db.ipynb) if you test with the download points. -This demo could only work with v23.02.0 version or previous. - -taxi-zones map: - - - -nyct2000 map: - - - -nyct-community-districts map: - - - -## Build -Firstly build the UDF JAR from source code before running this demo. -You can build the JAR [in Docker](#build-in-docker) with the provided [Dockerfile](Dockerfile), -or [in local machine](#build-in-local-machine) after prerequisites. - -### Build in Docker -1. Build the docker image [Dockerfile](Dockerfile), then run the container. - ```Bash - docker build -f Dockerfile . -t build-spark-cuspatial - docker run -it build-spark-cuspatial bash - ``` -2. Bash into the Docker and install libcuspatial - ```Bash - conda install -c rapidsai-nightly -c nvidia -c conda-forge -c defaults libcuspatial=23.02 - ``` -3. Get the code, then run `mvn package`. - ```Bash - git clone https://github.com/NVIDIA/spark-rapids-examples.git - cd spark-rapids-examples/examples/UDF-Examples/Spark-cuSpatial/ - mvn package - ``` -4. You'll get the jar named `spark-cuspatial-.jar` in the target folder. - -Note: The docker env is just for building the jar, not for running the application. - -### Build in local machine: -1. Essential build tools: - - [cmake(>=3.20)](https://cmake.org/download/), - - [ninja(>=1.10)](https://github.com/ninja-build/ninja/releases), - - [gcc(>=9.3)](https://gcc.gnu.org/releases.html) -2. [CUDA Toolkit(>=11.0)](https://developer.nvidia.com/cuda-toolkit) -3. conda: use [miniconda](https://docs.conda.io/en/latest/miniconda.html) to maintain header files and cmake dependecies -4. [cuspatial](https://github.com/rapidsai/cuspatial): install libcuspatial - ```Bash - # Install libcuspatial from conda - conda install -c rapidsai-nightly -c nvidia -c conda-forge -c defaults libcuspatial=23.02 - ``` -5. Build the JAR using `mvn package`. - ```Bash - git clone https://github.com/NVIDIA/spark-rapids-examples.git - cd spark-rapids-examples/examples/Spark-cuSpatial/ - mvn package - ``` -6. `spark-cuspatial-.jar` will be generated in the target folder. - -## Run -### GPU Demo on Spark Standalone on-premises cluster -1. Set up [a standalone cluster](../../../docs/get-started/xgboost-examples/on-prem-cluster/standalone-scala.md) of Spark. - Make sure the conda/lib is included in LD_LIBRARY_PATH, so that spark executors can load libcuspatial.so. - -2. Download Spark RAPIDS JAR - * [Spark RAPIDS JAR v23.02.0](https://repo1.maven.org/maven2/com/nvidia/rapids-4-spark_2.12/23.02.0/rapids-4-spark_2.12-23.02.0.jar) or above -3. Prepare sample dataset and JARs. Copy the [sample dataset](../../../datasets/cuspatial_data.tar.gz) to `/data/cuspatial_data/`. - Copy Spark RAPIDS JAR and `spark-cuspatial-.jar` to `/data/cuspatial_data/jars/`. - If you build the `spark-cuspatial-.jar` in docker, please copy the jar from docker to local: - ``` - docker cp YOUR_DOCKER_CONTAINER:/PATH/TO/spark-cuspatial-.jar ./YOUR_LOCAL_PATH - ``` - Note: update the paths in `gpu-run.sh` accordingly. -4. Run `gpu-run.sh` - ```Bash - ./gpu-run.sh - ``` -### GPU Demo on AWS Databricks -1. Build a customized docker image using [Dockerfile.awsdb](Dockerfile.awsdb) and push to a Docker registry such as [Docker Hub](https://hub.docker.com/) which can be accessible by AWS Databricks. - ```Bash - # replace your dockerhub repo, your tag or any other repo AWS DB can access - docker build -f Dockerfile.awsdb . -t : - docker push : - ``` - -2. Follow the [Spark-rapids get-started document](https://docs.nvidia.com/spark-rapids/user-guide/latest/getting-started/databricks.html) to create a GPU cluster on AWS Databricks. - Below are some different steps since a custom docker image is used with Databricks: - * Databricks Runtime Version - Choose a non-ML Databricks Runtime such as `Runtime: 9.1 LTS(Scala 2.12, Spark 3.1.2)` and - choose GPU AWS instance type such as `g4dn.xlarge`. Note that ML runtime does not support customized docker container with below messages: -`Support for Databricks container services requires runtime version 5.3+` - and the `Confirm` button is disabled. - * Use your own Docker container - Input `Docker Image URL` as `your-dockerhub-repo:your-tag` - * Follow the Databricks get-started document for other steps. - -3. Copy the sample [cuspatial_data.tar.gz](../../../datasets/cuspatial_data.tar.gz) or your data to DBFS by using Databricks CLI. - ```Bash - # extract the data - tar zxf cuspatial_data.tar.gz - databricks fs cp -r cuspatial_data/* dbfs:/data/cuspatial_data/ - databricks fs ls dbfs:/data/cuspatial_data/ - # it should have below 2 folders. - points - polygons - ``` - The sample points and polygons are randomly generated. - - Sample polygons: - - - -4. Upload `spark-cuspatial-.jar` on dbfs and then install it in Databricks cluster. - - - -5. Import [cuspatial_sample.ipynb](notebooks/cuspatial_sample_db.ipynb) to Databricks workspace, attach it to Databricks cluster and run it. - -### CPU Demo on AWS Databricks -1. Create a Databricks cluster. For example, Databricks Runtime 10.3. - -2. Install the Sedona jars and Sedona Python libs on Databricks using web UI. - The Sedona version should be 1.1.1-incubating or higher. - * Install below jars from Maven Coordinates in Libraries tab: - ```Bash - org.apache.sedona:sedona-python-adapter-3.0_2.12:1.2.0-incubating - org.datasyslab:geotools-wrapper:1.1.0-25.2 - ``` - * To enable python support, install below python lib from PyPI in Libraries tab - ```Bash - apache-sedona - ``` -3. From your cluster configuration (Cluster -> Edit -> Configuration -> Advanced options -> Spark) activate the - Sedona functions and the kryo serializer by adding below to the Spark Config - ```Bash - spark.sql.extensions org.apache.sedona.viz.sql.SedonaVizExtensions,org.apache.sedona.sql.SedonaSqlExtensions - spark.serializer org.apache.spark.serializer.KryoSerializer - spark.kryo.registrator org.apache.sedona.core.serde.SedonaKryoRegistrator - ``` - -4. Upload the sample data files to DBFS, start the cluster, attach the [notebook](notebooks/spacial-cpu-apache-sedona_db.ipynb) to the cluster, and run it. diff --git a/examples/UDF-Examples/Spark-cuSpatial/gpu-run.sh b/examples/UDF-Examples/Spark-cuSpatial/gpu-run.sh deleted file mode 100755 index c79947967..000000000 --- a/examples/UDF-Examples/Spark-cuSpatial/gpu-run.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/bash -# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# change to your spark folder -SPARK_HOME=${SPARK_HOME:-/data/spark-3.2.0-bin-hadoop3.2} - -# change this path to your root path for the dataset -ROOT_PATH=${ROOT_PATH:-/data/cuspatial_data} -# Extract the sample dataset in ../../datasets/cuspatial_data.tar.gz -# Copy the polygons and points data into the root path or change the root path to where they are -SHAPE_FILE_DIR=$ROOT_PATH/polygons -SHAPE_FILE_NAME="polygons" -DATA_IN_PATH=$ROOT_PATH/points -DATA_OUT_PATH=$ROOT_PATH/output - -rm -rf $DATA_OUT_PATH - -# the path to keep the jars of spark-rapids & spark-cuspatial -JARS=$ROOT_PATH/jars - -JARS_PATH=${JARS_PATH:-$JARS/rapids-4-spark_2.12-23.02.0.jar,$JARS/spark-cuspatial-23.02.0.jar} - -$SPARK_HOME/bin/spark-submit --master spark://$HOSTNAME:7077 \ ---name "Gpu Spatial Join UDF" \ ---executor-memory 20G \ ---executor-cores 10 \ ---conf spark.task.cpus=1 \ ---conf spark.sql.adaptive.enabled=false \ ---conf spark.plugins=com.nvidia.spark.SQLPlugin \ ---conf spark.rapids.sql.explain=all \ ---conf spark.executor.resource.gpu.amount=1 \ ---conf spark.cuspatial.sql.udf.shapeFileName="$SHAPE_FILE_NAME.shp" \ ---conf spark.driver.extraLibraryPath=YOUR_LD_LIBRARY_PATH \ ---conf spark.executor.extraLibraryPath=YOUR_LD_LIBRARY_PATH \ ---jars $JARS_PATH \ ---files $SHAPE_FILE_DIR/$SHAPE_FILE_NAME.shp,$SHAPE_FILE_DIR/$SHAPE_FILE_NAME.shx \ -spatial_join.py $DATA_IN_PATH $DATA_OUT_PATH diff --git a/examples/UDF-Examples/Spark-cuSpatial/notebooks/cuspatial_sample_db.ipynb b/examples/UDF-Examples/Spark-cuSpatial/notebooks/cuspatial_sample_db.ipynb deleted file mode 100644 index 4797ada9c..000000000 --- a/examples/UDF-Examples/Spark-cuSpatial/notebooks/cuspatial_sample_db.ipynb +++ /dev/null @@ -1,328 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "inputWidgets": {}, - "nuid": "4f12a9eb-6580-4575-a6a8-4c08820fe8e0", - "showTitle": false, - "title": "" - } - }, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "
" - ] - }, - "metadata": { - "application/vnd.databricks.v1+output": { - "addedWidgets": {}, - "arguments": {}, - "data": "
", - "datasetInfos": [], - "metadata": {}, - "removedWidgets": [], - "type": "html" - } - }, - "output_type": "display_data" - } - ], - "source": [ - "#define the input & output path\n", - "inputPath='dbfs:/data/cuspatial_data/points'\n", - "outputPath='dbfs:/data/output'\n", - "\n", - "# add the shapefile(.shp & .shx)\n", - "spark.sparkContext.addFile(\"dbfs:/data/cuspatial_data/polygons/polygons.shp\")\n", - "spark.sparkContext.addFile(\"dbfs:/data/cuspatial_data/polygons/polygons.shx\")\n", - "spark.conf.set(\"spark.cuspatial.sql.udf.shapeFileName\", \"polygons.shp\")" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "inputWidgets": {}, - "nuid": "741761ee-d5cf-4c43-ae73-30703d246901", - "showTitle": false, - "title": "" - } - }, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "
" - ] - }, - "metadata": { - "application/vnd.databricks.v1+output": { - "addedWidgets": {}, - "arguments": {}, - "data": "
", - "datasetInfos": [], - "metadata": {}, - "removedWidgets": [], - "type": "html" - } - }, - "output_type": "display_data" - } - ], - "source": [ - "# register the UDF\n", - "spark.udf.registerJavaFunction(\"point_in_polygon\", \"com.nvidia.spark.rapids.udf.PointInPolygon\", None)\n", - "\n", - "df = spark.read.parquet(inputPath)\n", - "df = df.selectExpr('x', 'y', 'point_in_polygon(x, y) as ret')\n", - "df.write.mode(\"overwrite\").parquet(outputPath)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "inputWidgets": {}, - "nuid": "592bfcde-ca6f-4b94-aac1-e3b298f25fcb", - "showTitle": false, - "title": "" - } - }, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "
Input rows: 71\n", - "
" - ] - }, - "metadata": { - "application/vnd.databricks.v1+output": { - "addedWidgets": {}, - "arguments": {}, - "data": "
Input rows: 71\n
", - "datasetInfos": [], - "metadata": {}, - "removedWidgets": [], - "type": "html" - } - }, - "output_type": "display_data" - } - ], - "source": [ - "print(\"Input rows: \", df.count())" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "inputWidgets": {}, - "nuid": "f1b744a4-13a3-4260-b8f0-efcbbf1c0337", - "showTitle": false, - "title": "" - } - }, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "
Output rows: 71\n", - "+-------------------+--------------------+---+\n", - " x| y|ret|\n", - "+-------------------+--------------------+---+\n", - "0.48171647380517046| 1.9022922214961997| []|\n", - " 1.2591725716781235| 0.1448705855995005| []|\n", - " 0.1895259128530169| 0.5431061133894604| []|\n", - " 3.028362149164369|0.027638405909631958| []|\n", - " 1.3890664414691907| 1.5177694304735412| []|\n", - " 3.1907684812039956| 0.2621847215928189| []|\n", - " 3.7080407833612004| 0.09804238103130436| []|\n", - " 3.0706987088385853| 0.9376313558467103| []|\n", - " 2.0697434332621234| 1.1809465376402173| []|\n", - " 2.175448214220591| 1.2372448404986038| []|\n", - " 2.113652420701984| 1.2774712415624014| []|\n", - " 2.9909779614491687| 1.2420487904041893| []|\n", - " 4.07037627210835| 1.9486902798139454| []|\n", - " 4.822583857757069| 0.3234041700489503| []|\n", - " 4.849847745942472| 1.9531893897409585| []|\n", - " 4.732546857961497| 0.5659923375279095| []|\n", - " 4.529792124514895| 1.942673409259531| []|\n", - " 3.2648444465931474| 2.693039435509084| []|\n", - " 2.1807636574967466| 3.2296461832828114|[3]|\n", - " 3.7164018490892348| 2.4612194182614333| []|\n", - " 2.2006520196663066| 3.7672478678985257|[3]|\n", - " 6.291790729917634| 2.983311357415729|[1]|\n", - " 2.5104987015171574| 3.0668114607133137|[3]|\n", - " 2.3007438625108882| 3.6045900851589048|[3]|\n", - " 6.101327777646798| 2.5239201807166616|[1]|\n", - " 6.109985464455084| 2.2235950639628523|[1]|\n", - " 6.4274219368674315| 2.9754616970668213|[1]|\n", - " 7.886010001346151| 3.538128217886674| []|\n", - " 7.5085184104988| 3.623862886287816| []|\n", - " 7.430677191305505| 3.380489849365283| []|\n", - " 1.7015273093278767| 7.478882372510933| []|\n", - " 7.769497359206111| 3.253257011908445| []|\n", - " 3.86008672302403| 7.513564222799629| []|\n", - " 0.059011873032214| 5.823535317960799| []|\n", - " 3.154282922203257| 5.788316610960881| []|\n", - " 2.4264509160270813| 5.188939408363776|[0]|\n", - " 1.9804558865545805| 1.3472225743317712| []|\n", - " 0.8178039499335275| 0.8138440641113271| []|\n", - " 0.2536015260915061| 1.8762161698642947| []|\n", - " 3.710910700915217| 0.9937713340192049| []|\n", - " 3.918090468102582| 0.3338651960183463| []|\n", - " 3.572744183805594| 0.33184908855075124| []|\n", - " 3.70669993057843| 0.7485845679979923| []|\n", - " 3.3588457228653024| 0.2346381514128677| []|\n", - " 2.520755151373394| 1.902015274420646| []|\n", - " 2.5322042870739683| 1.419555755682142| []|\n", - " 2.4613232527836137| 1.0484414482621331| []|\n", - " 4.975578758530645| 0.9606291981013242| []|\n", - " 4.5584381091040616| 1.8996548860019926| []|\n", - " 4.300706849071861|0.021365525588281198| []|\n", - " 3.01954722322135| 2.57810040095543| []|\n", - " 3.7622247877537456| 2.8709552313924487| []|\n", - " 4.75489831780737| 0.7800065259479418| []|\n", - " 2.566986568683904| 3.6607732238530897|[3]|\n", - " 3.7002781846945347| 2.3345952955903906| []|\n", - " 2.493975723955388| 3.3999020934055837|[3]|\n", - " 2.8222482218882474| 3.8159308233351266|[3]|\n", - " 6.0821276168848994| 2.5470532680258002|[1]|\n", - " 2.241538022180476| 3.8812819070357545|[3]|\n", - " 6.325158445513714| 2.8765450351723674|[1]|\n", - " 6.444584786789386| 2.174562817047202|[1]|\n", - " 6.6793884701899| 2.5605928243991434|[1]|\n", - " 7.250745898479374| 3.4154469467473447| []|\n", - " 7.079453687660189| 3.063690547962938|[1]|\n", - " 7.897735998643542| 3.380784914178574| []|\n", - " 2.2065031771469| 6.896038613284851| []|\n", - " 1.8703303641352362| 4.209727933188015|[3]|\n", - " 2.7456295127617385| 7.474216636277054| []|\n", - " 1.9143371250907073| 6.885401350515916| []|\n", - " 3.7176098065039747| 6.194330707468438| []|\n", - " 3.1162712022943757| 6.789029097334483| []|\n", - "+-------------------+--------------------+---+\n", - "\n", - "
" - ] - }, - "metadata": { - "application/vnd.databricks.v1+output": { - "addedWidgets": {}, - "arguments": {}, - "data": "
Output rows: 71\n+-------------------+--------------------+---+\n| x| y|ret|\n+-------------------+--------------------+---+\n|0.48171647380517046| 1.9022922214961997| []|\n| 1.2591725716781235| 0.1448705855995005| []|\n| 0.1895259128530169| 0.5431061133894604| []|\n| 3.028362149164369|0.027638405909631958| []|\n| 1.3890664414691907| 1.5177694304735412| []|\n| 3.1907684812039956| 0.2621847215928189| []|\n| 3.7080407833612004| 0.09804238103130436| []|\n| 3.0706987088385853| 0.9376313558467103| []|\n| 2.0697434332621234| 1.1809465376402173| []|\n| 2.175448214220591| 1.2372448404986038| []|\n| 2.113652420701984| 1.2774712415624014| []|\n| 2.9909779614491687| 1.2420487904041893| []|\n| 4.07037627210835| 1.9486902798139454| []|\n| 4.822583857757069| 0.3234041700489503| []|\n| 4.849847745942472| 1.9531893897409585| []|\n| 4.732546857961497| 0.5659923375279095| []|\n| 4.529792124514895| 1.942673409259531| []|\n| 3.2648444465931474| 2.693039435509084| []|\n| 2.1807636574967466| 3.2296461832828114|[3]|\n| 3.7164018490892348| 2.4612194182614333| []|\n| 2.2006520196663066| 3.7672478678985257|[3]|\n| 6.291790729917634| 2.983311357415729|[1]|\n| 2.5104987015171574| 3.0668114607133137|[3]|\n| 2.3007438625108882| 3.6045900851589048|[3]|\n| 6.101327777646798| 2.5239201807166616|[1]|\n| 6.109985464455084| 2.2235950639628523|[1]|\n| 6.4274219368674315| 2.9754616970668213|[1]|\n| 7.886010001346151| 3.538128217886674| []|\n| 7.5085184104988| 3.623862886287816| []|\n| 7.430677191305505| 3.380489849365283| []|\n| 1.7015273093278767| 7.478882372510933| []|\n| 7.769497359206111| 3.253257011908445| []|\n| 3.86008672302403| 7.513564222799629| []|\n| 0.059011873032214| 5.823535317960799| []|\n| 3.154282922203257| 5.788316610960881| []|\n| 2.4264509160270813| 5.188939408363776|[0]|\n| 1.9804558865545805| 1.3472225743317712| []|\n| 0.8178039499335275| 0.8138440641113271| []|\n| 0.2536015260915061| 1.8762161698642947| []|\n| 3.710910700915217| 0.9937713340192049| []|\n| 3.918090468102582| 0.3338651960183463| []|\n| 3.572744183805594| 0.33184908855075124| []|\n| 3.70669993057843| 0.7485845679979923| []|\n| 3.3588457228653024| 0.2346381514128677| []|\n| 2.520755151373394| 1.902015274420646| []|\n| 2.5322042870739683| 1.419555755682142| []|\n| 2.4613232527836137| 1.0484414482621331| []|\n| 4.975578758530645| 0.9606291981013242| []|\n| 4.5584381091040616| 1.8996548860019926| []|\n| 4.300706849071861|0.021365525588281198| []|\n| 3.01954722322135| 2.57810040095543| []|\n| 3.7622247877537456| 2.8709552313924487| []|\n| 4.75489831780737| 0.7800065259479418| []|\n| 2.566986568683904| 3.6607732238530897|[3]|\n| 3.7002781846945347| 2.3345952955903906| []|\n| 2.493975723955388| 3.3999020934055837|[3]|\n| 2.8222482218882474| 3.8159308233351266|[3]|\n| 6.0821276168848994| 2.5470532680258002|[1]|\n| 2.241538022180476| 3.8812819070357545|[3]|\n| 6.325158445513714| 2.8765450351723674|[1]|\n| 6.444584786789386| 2.174562817047202|[1]|\n| 6.6793884701899| 2.5605928243991434|[1]|\n| 7.250745898479374| 3.4154469467473447| []|\n| 7.079453687660189| 3.063690547962938|[1]|\n| 7.897735998643542| 3.380784914178574| []|\n| 2.2065031771469| 6.896038613284851| []|\n| 1.8703303641352362| 4.209727933188015|[3]|\n| 2.7456295127617385| 7.474216636277054| []|\n| 1.9143371250907073| 6.885401350515916| []|\n| 3.7176098065039747| 6.194330707468438| []|\n| 3.1162712022943757| 6.789029097334483| []|\n+-------------------+--------------------+---+\n\n
", - "datasetInfos": [], - "metadata": {}, - "removedWidgets": [], - "type": "html" - } - }, - "output_type": "display_data" - } - ], - "source": [ - "# show the result\n", - "result = spark.read.parquet(outputPath)\n", - "print(\"Output rows: \", result.count())\n", - "result.show(71)" - ] - } - ], - "metadata": { - "application/vnd.databricks.v1+notebook": { - "dashboards": [], - "language": "python", - "notebookMetadata": { - "pythonIndentUnit": 2 - }, - "notebookName": "cuspatial_sample", - "notebookOrigID": 3352849421916703, - "widgets": {} - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.2" - } - }, - "nbformat": 4, - "nbformat_minor": 1 -} diff --git a/examples/UDF-Examples/Spark-cuSpatial/notebooks/cuspatial_sample_standalone.ipynb b/examples/UDF-Examples/Spark-cuSpatial/notebooks/cuspatial_sample_standalone.ipynb deleted file mode 100644 index 7c2a13477..000000000 --- a/examples/UDF-Examples/Spark-cuSpatial/notebooks/cuspatial_sample_standalone.ipynb +++ /dev/null @@ -1,100 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "id": "26b29706", - "metadata": {}, - "outputs": [], - "source": [ - "from pyspark.sql import SparkSession\n", - "import os\n", - "jarsPath = os.getenv(\"JARS_PATH\", \"/data/cuspatial_data/jars/rapids-4-spark_2.12-23.02.0.jar,/data/cuspatial_data/jars/spark-cuspatial-23.02.0.jar\")\n", - "spark = SparkSession.builder \\\n", - " .config(\"spark.jars\", jarsPath) \\\n", - " .config(\"spark.sql.adaptive.enabled\", \"false\") \\\n", - " .config(\"spark.executor.memory\", \"20GB\") \\\n", - " .config(\"spark.executor.cores\", \"6\") \\\n", - " .config(\"spark.plugins\", \"com.nvidia.spark.SQLPlugin\") \\\n", - " .config(\"spark.executor.resource.gpu.amount\", \"1\") \\\n", - " .getOrCreate()\n", - "# register the udf\n", - "spark.udf.registerJavaFunction(\"point_in_polygon\", \"com.nvidia.spark.rapids.udf.PointInPolygon\", None)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "cce700a8", - "metadata": {}, - "outputs": [], - "source": [ - "# prepare shape files\n", - "rootPath = os.getenv(\"ROOT_PATH\", \"/data/cuspatial_data\")\n", - "spark.sparkContext.addFile(rootPath + \"/polygons/polygons.shp\")\n", - "spark.sparkContext.addFile(rootPath + \"/polygons/polygons.shx\")\n", - "\n", - "inputPath = rootPath + \"/points/\"\n", - "outputPath = rootPath + \"/output/\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "cd26c441", - "metadata": {}, - "outputs": [], - "source": [ - "# set its parameters via SQL config for runtime updating\n", - "spark.conf.set(\"spark.cuspatial.sql.udf.shapeFileName\", \"polygons.shp\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6b1016ce", - "metadata": {}, - "outputs": [], - "source": [ - "# read the points data\n", - "df = spark.read.parquet(inputPath)\n", - "df = df.selectExpr('x', 'y', 'point_in_polygon(x, y) as point_in_polygon')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "09d5e22e", - "metadata": {}, - "outputs": [], - "source": [ - "import time\n", - "begin = time.time()\n", - "df.write.mode(\"overwrite\").parquet(outputPath)\n", - "end = time.time()\n", - "print(\"==> It took {} s\".format(round(end-begin, 2)))" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.12" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/examples/UDF-Examples/Spark-cuSpatial/notebooks/spacial-cpu-apache-sedona_db.ipynb b/examples/UDF-Examples/Spark-cuSpatial/notebooks/spacial-cpu-apache-sedona_db.ipynb deleted file mode 100644 index 47cf5e872..000000000 --- a/examples/UDF-Examples/Spark-cuSpatial/notebooks/spacial-cpu-apache-sedona_db.ipynb +++ /dev/null @@ -1,364 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "inputWidgets": {}, - "nuid": "61a74001-716b-4411-aecb-77d07058d200", - "showTitle": false, - "title": "" - } - }, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "
Requirement already satisfied: geopandas in /databricks/python3/lib/python3.8/site-packages (0.11.0)\n", - "Requirement already satisfied: pyproj>=2.6.1.post1 in /databricks/python3/lib/python3.8/site-packages (from geopandas) (3.3.1)\n", - "Requirement already satisfied: shapely<2,>=1.7 in /databricks/python3/lib/python3.8/site-packages (from geopandas) (1.8.2)\n", - "Requirement already satisfied: pandas>=1.0.0 in /databricks/python3/lib/python3.8/site-packages (from geopandas) (1.2.4)\n", - "Requirement already satisfied: packaging in /databricks/python3/lib/python3.8/site-packages (from geopandas) (20.9)\n", - "Requirement already satisfied: fiona>=1.8 in /databricks/python3/lib/python3.8/site-packages (from geopandas) (1.8.21)\n", - "Requirement already satisfied: six>=1.7 in /databricks/python3/lib/python3.8/site-packages (from fiona>=1.8->geopandas) (1.15.0)\n", - "Requirement already satisfied: setuptools in /usr/local/lib/python3.8/dist-packages (from fiona>=1.8->geopandas) (52.0.0)\n", - "Requirement already satisfied: click>=4.0 in /databricks/python3/lib/python3.8/site-packages (from fiona>=1.8->geopandas) (8.1.3)\n", - "Requirement already satisfied: certifi in /databricks/python3/lib/python3.8/site-packages (from fiona>=1.8->geopandas) (2020.12.5)\n", - "Requirement already satisfied: munch in /databricks/python3/lib/python3.8/site-packages (from fiona>=1.8->geopandas) (2.5.0)\n", - "Requirement already satisfied: cligj>=0.5 in /databricks/python3/lib/python3.8/site-packages (from fiona>=1.8->geopandas) (0.7.2)\n", - "Requirement already satisfied: click-plugins>=1.0 in /databricks/python3/lib/python3.8/site-packages (from fiona>=1.8->geopandas) (1.1.1)\n", - "Requirement already satisfied: attrs>=17 in /databricks/python3/lib/python3.8/site-packages (from fiona>=1.8->geopandas) (20.3.0)\n", - "Requirement already satisfied: pytz>=2017.3 in /databricks/python3/lib/python3.8/site-packages (from pandas>=1.0.0->geopandas) (2020.5)\n", - "Requirement already satisfied: numpy>=1.16.5 in /databricks/python3/lib/python3.8/site-packages (from pandas>=1.0.0->geopandas) (1.20.1)\n", - "Requirement already satisfied: python-dateutil>=2.7.3 in /databricks/python3/lib/python3.8/site-packages (from pandas>=1.0.0->geopandas) (2.8.1)\n", - "Requirement already satisfied: pyparsing>=2.0.2 in /databricks/python3/lib/python3.8/site-packages (from packaging->geopandas) (2.4.7)\n", - "WARNING: You are using pip version 21.0.1; however, version 22.1.2 is available.\n", - "You should consider upgrading via the '/databricks/python3/bin/python -m pip install --upgrade pip' command.\n", - "
" - ] - }, - "metadata": { - "application/vnd.databricks.v1+output": { - "addedWidgets": {}, - "arguments": {}, - "data": "
Requirement already satisfied: geopandas in /databricks/python3/lib/python3.8/site-packages (0.11.0)\nRequirement already satisfied: pyproj>=2.6.1.post1 in /databricks/python3/lib/python3.8/site-packages (from geopandas) (3.3.1)\nRequirement already satisfied: shapely<2,>=1.7 in /databricks/python3/lib/python3.8/site-packages (from geopandas) (1.8.2)\nRequirement already satisfied: pandas>=1.0.0 in /databricks/python3/lib/python3.8/site-packages (from geopandas) (1.2.4)\nRequirement already satisfied: packaging in /databricks/python3/lib/python3.8/site-packages (from geopandas) (20.9)\nRequirement already satisfied: fiona>=1.8 in /databricks/python3/lib/python3.8/site-packages (from geopandas) (1.8.21)\nRequirement already satisfied: six>=1.7 in /databricks/python3/lib/python3.8/site-packages (from fiona>=1.8->geopandas) (1.15.0)\nRequirement already satisfied: setuptools in /usr/local/lib/python3.8/dist-packages (from fiona>=1.8->geopandas) (52.0.0)\nRequirement already satisfied: click>=4.0 in /databricks/python3/lib/python3.8/site-packages (from fiona>=1.8->geopandas) (8.1.3)\nRequirement already satisfied: certifi in /databricks/python3/lib/python3.8/site-packages (from fiona>=1.8->geopandas) (2020.12.5)\nRequirement already satisfied: munch in /databricks/python3/lib/python3.8/site-packages (from fiona>=1.8->geopandas) (2.5.0)\nRequirement already satisfied: cligj>=0.5 in /databricks/python3/lib/python3.8/site-packages (from fiona>=1.8->geopandas) (0.7.2)\nRequirement already satisfied: click-plugins>=1.0 in /databricks/python3/lib/python3.8/site-packages (from fiona>=1.8->geopandas) (1.1.1)\nRequirement already satisfied: attrs>=17 in /databricks/python3/lib/python3.8/site-packages (from fiona>=1.8->geopandas) (20.3.0)\nRequirement already satisfied: pytz>=2017.3 in /databricks/python3/lib/python3.8/site-packages (from pandas>=1.0.0->geopandas) (2020.5)\nRequirement already satisfied: numpy>=1.16.5 in /databricks/python3/lib/python3.8/site-packages (from pandas>=1.0.0->geopandas) (1.20.1)\nRequirement already satisfied: python-dateutil>=2.7.3 in /databricks/python3/lib/python3.8/site-packages (from pandas>=1.0.0->geopandas) (2.8.1)\nRequirement already satisfied: pyparsing>=2.0.2 in /databricks/python3/lib/python3.8/site-packages (from packaging->geopandas) (2.4.7)\nWARNING: You are using pip version 21.0.1; however, version 22.1.2 is available.\nYou should consider upgrading via the '/databricks/python3/bin/python -m pip install --upgrade pip' command.\n
", - "datasetInfos": [], - "metadata": {}, - "removedWidgets": [], - "type": "html" - } - }, - "output_type": "display_data" - } - ], - "source": [ - "%sh \n", - "pip install geopandas" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "inputWidgets": {}, - "nuid": "4fd97bf3-dc08-495f-9cfe-e9e551f40e16", - "showTitle": false, - "title": "" - } - }, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "
" - ] - }, - "metadata": { - "application/vnd.databricks.v1+output": { - "addedWidgets": {}, - "arguments": {}, - "data": "
", - "datasetInfos": [], - "metadata": {}, - "removedWidgets": [], - "type": "html" - } - }, - "output_type": "display_data" - } - ], - "source": [ - "import os\n", - "import geopandas as gpd\n", - "from pyspark.sql.functions import col, expr, when\n", - "from sedona.register import SedonaRegistrator\n", - "from sedona.utils import SedonaKryoRegistrator, KryoSerializer" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "inputWidgets": {}, - "nuid": "6b15de34-d411-457b-89fb-7232587ae949", - "showTitle": false, - "title": "" - } - }, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "
/databricks/spark/python/pyspark/sql/pandas/conversion.py:340: UserWarning: createDataFrame attempted Arrow optimization because 'spark.sql.execution.arrow.pyspark.enabled' is set to true; however, failed by the reason below:\n", - " Did not pass numpy.dtype object\n", - "Attempting non-optimization as 'spark.sql.execution.arrow.pyspark.fallback.enabled' is set to true.\n", - " warnings.warn(msg)\n", - "Out[9]: 4
" - ] - }, - "metadata": { - "application/vnd.databricks.v1+output": { - "addedWidgets": {}, - "arguments": {}, - "data": "
/databricks/spark/python/pyspark/sql/pandas/conversion.py:340: UserWarning: createDataFrame attempted Arrow optimization because 'spark.sql.execution.arrow.pyspark.enabled' is set to true; however, failed by the reason below:\n Did not pass numpy.dtype object\nAttempting non-optimization as 'spark.sql.execution.arrow.pyspark.fallback.enabled' is set to true.\n warnings.warn(msg)\nOut[9]: 4
", - "datasetInfos": [], - "metadata": {}, - "removedWidgets": [], - "type": "html" - } - }, - "output_type": "display_data" - } - ], - "source": [ - "# read the shapefile\n", - "polygons = gpd.read_file(\"/dbfs/data/cuspatial_data/polygons/polygons.shp\")\n", - "polygons_df = spark.createDataFrame(\n", - " polygons\n", - ")\n", - "polygons_df.createOrReplaceTempView(\"polygons\")\n", - "polygons_df.count()" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "inputWidgets": {}, - "nuid": "3a4d1a80-72e4-490d-8152-f6f231cac37f", - "showTitle": false, - "title": "" - } - }, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "
+--------------------+\n", - " mypoint|\n", - "+--------------------+\n", - "POINT (0.48171647...|\n", - "POINT (1.25917257...|\n", - "POINT (0.18952591...|\n", - "POINT (3.02836214...|\n", - "POINT (1.38906644...|\n", - "+--------------------+\n", - "only showing top 5 rows\n", - "\n", - "
" - ] - }, - "metadata": { - "application/vnd.databricks.v1+output": { - "addedWidgets": {}, - "arguments": {}, - "data": "
+--------------------+\n| mypoint|\n+--------------------+\n|POINT (0.48171647...|\n|POINT (1.25917257...|\n|POINT (0.18952591...|\n|POINT (3.02836214...|\n|POINT (1.38906644...|\n+--------------------+\nonly showing top 5 rows\n\n
", - "datasetInfos": [], - "metadata": {}, - "removedWidgets": [], - "type": "html" - } - }, - "output_type": "display_data" - } - ], - "source": [ - "point_parquet_df = spark.read.format(\"parquet\").\\\n", - " load(\"dbfs:/data/cuspatial_data/points\")\n", - "\n", - "point_parquet_df.createOrReplaceTempView(\"pointtable\")\n", - "point_df = spark.sql(\"select ST_Point(x, y) as mypoint from pointtable\")\n", - "point_df.show(5)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "inputWidgets": {}, - "nuid": "8ec977f5-937e-45ce-89d6-46fa3b48cc39", - "showTitle": false, - "title": "" - } - }, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "
+--------------------+------------------+------------------+\n", - " geometry| x| y|\n", - "+--------------------+------------------+------------------+\n", - "POLYGON ((2.08811...|2.1807636574967466|3.2296461832828114|\n", - "POLYGON ((2.08811...|2.2006520196663066|3.7672478678985257|\n", - "POLYGON ((2.08811...|2.5104987015171574|3.0668114607133137|\n", - "POLYGON ((2.08811...|2.3007438625108882|3.6045900851589048|\n", - "POLYGON ((2.08811...| 2.566986568683904|3.6607732238530897|\n", - "POLYGON ((2.08811...| 2.493975723955388|3.3999020934055837|\n", - "POLYGON ((2.08811...|2.8222482218882474|3.8159308233351266|\n", - "POLYGON ((2.08811...| 2.241538022180476|3.8812819070357545|\n", - "POLYGON ((2.08811...|1.8703303641352362| 4.209727933188015|\n", - "POLYGON ((2.48845...|2.4264509160270813| 5.188939408363776|\n", - "POLYGON ((5.03982...| 6.291790729917634| 2.983311357415729|\n", - "POLYGON ((5.03982...| 6.101327777646798|2.5239201807166616|\n", - "POLYGON ((5.03982...| 6.109985464455084|2.2235950639628523|\n", - "POLYGON ((5.03982...|6.4274219368674315|2.9754616970668213|\n", - "POLYGON ((5.03982...|6.0821276168848994|2.5470532680258002|\n", - "POLYGON ((5.03982...| 6.325158445513714|2.8765450351723674|\n", - "POLYGON ((5.03982...| 6.444584786789386| 2.174562817047202|\n", - "POLYGON ((5.03982...| 6.6793884701899|2.5605928243991434|\n", - "POLYGON ((5.03982...| 7.079453687660189| 3.063690547962938|\n", - "+--------------------+------------------+------------------+\n", - "\n", - "
" - ] - }, - "metadata": { - "application/vnd.databricks.v1+output": { - "addedWidgets": {}, - "arguments": {}, - "data": "
+--------------------+------------------+------------------+\n| geometry| x| y|\n+--------------------+------------------+------------------+\n|POLYGON ((2.08811...|2.1807636574967466|3.2296461832828114|\n|POLYGON ((2.08811...|2.2006520196663066|3.7672478678985257|\n|POLYGON ((2.08811...|2.5104987015171574|3.0668114607133137|\n|POLYGON ((2.08811...|2.3007438625108882|3.6045900851589048|\n|POLYGON ((2.08811...| 2.566986568683904|3.6607732238530897|\n|POLYGON ((2.08811...| 2.493975723955388|3.3999020934055837|\n|POLYGON ((2.08811...|2.8222482218882474|3.8159308233351266|\n|POLYGON ((2.08811...| 2.241538022180476|3.8812819070357545|\n|POLYGON ((2.08811...|1.8703303641352362| 4.209727933188015|\n|POLYGON ((2.48845...|2.4264509160270813| 5.188939408363776|\n|POLYGON ((5.03982...| 6.291790729917634| 2.983311357415729|\n|POLYGON ((5.03982...| 6.101327777646798|2.5239201807166616|\n|POLYGON ((5.03982...| 6.109985464455084|2.2235950639628523|\n|POLYGON ((5.03982...|6.4274219368674315|2.9754616970668213|\n|POLYGON ((5.03982...|6.0821276168848994|2.5470532680258002|\n|POLYGON ((5.03982...| 6.325158445513714|2.8765450351723674|\n|POLYGON ((5.03982...| 6.444584786789386| 2.174562817047202|\n|POLYGON ((5.03982...| 6.6793884701899|2.5605928243991434|\n|POLYGON ((5.03982...| 7.079453687660189| 3.063690547962938|\n+--------------------+------------------+------------------+\n\n
", - "datasetInfos": [], - "metadata": {}, - "removedWidgets": [], - "type": "html" - } - }, - "output_type": "display_data" - } - ], - "source": [ - "result = spark.sql(\"\\\n", - "SELECT polygons.*, pointtable.* FROM polygons ,pointtable \\\n", - "WHERE ST_Contains(polygons.geometry, ST_Point(pointtable.x, pointtable.y)) \\\n", - "\")\n", - "result.show()" - ] - } - ], - "metadata": { - "application/vnd.databricks.v1+notebook": { - "dashboards": [], - "language": "python", - "notebookMetadata": { - "pythonIndentUnit": 2 - }, - "notebookName": "spacial-cpu-apache-sedona", - "notebookOrigID": 1618423020047086, - "widgets": {} - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.2" - } - }, - "nbformat": 4, - "nbformat_minor": 1 -} diff --git a/examples/UDF-Examples/Spark-cuSpatial/pom.xml b/examples/UDF-Examples/Spark-cuSpatial/pom.xml deleted file mode 100644 index 17bc2c205..000000000 --- a/examples/UDF-Examples/Spark-cuSpatial/pom.xml +++ /dev/null @@ -1,140 +0,0 @@ - - - - 4.0.0 - - com.nvidia - spark-cuspatial - UDF of the cuSpatial case for the RAPIDS Accelerator - The RAPIDS accelerated user defined function of the cuSpatial case - for use with the RAPIDS Accelerator for Apache Spark - 23.02.0-SNAPSHOT - - - 1.8 - 1.8 - 8 - - 23.02.0 - 2.12 - 3.2.0 - ${project.build.directory}/cpp-build - - 10 - - - - - com.nvidia - rapids-4-spark_${scala.binary.version} - ${rapids.version} - provided - - - org.apache.spark - spark-sql_${scala.binary.version} - ${spark.version} - provided - - - - - - - ${project.build.directory}/native-deps/ - - - - - maven-antrun-plugin - 3.0.0 - - - cmake - validate - - - - - - - - - - - - - - - - - run - - - - - - maven-resources-plugin - 3.2.0 - - - copy-native-libs - validate - - copy-resources - - - true - ${project.build.directory}/native-deps/${os.arch}/${os.name} - - - ${udf.native.build.path} - - libspatialudfjni.so - - - - - - - - - - - - - assembly-udf-jar - - - - maven-assembly-plugin - - - jar-with-dependencies - - - - - - - - diff --git a/examples/UDF-Examples/Spark-cuSpatial/spatial_join.py b/examples/UDF-Examples/Spark-cuSpatial/spatial_join.py deleted file mode 100644 index a36a0b865..000000000 --- a/examples/UDF-Examples/Spark-cuSpatial/spatial_join.py +++ /dev/null @@ -1,47 +0,0 @@ -# -# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import sys -import time -from pyspark.sql import SparkSession - - -if __name__ == '__main__': - if len(sys.argv) < 2: - raise Exception("Requires a data path.") - inputPath = sys.argv[1] - outputPath = sys.argv[2] - - # You should set correct shapefile name by "--conf spark.cuspatial.sql.udf.shapeFileName" - spark = SparkSession.builder.getOrCreate() - # register the udf and set its parameters via the runtime config - spark.udf.registerJavaFunction("point_in_polygon", "com.nvidia.spark.rapids.udf.PointInPolygon", None) - - # read the points data - df = spark.read.parquet(inputPath) - # null row is not supported yet by the UDF, filter out them first. - df = df.filter("x is not NULL and y is not NULL") - - # test func start - df = df.selectExpr('x', 'y', 'point_in_polygon(x, y) as point_in_polygon') - # test func end - - # trigger the test - begin = time.time() - df.write.mode("overwrite").parquet(outputPath) - end = time.time() - print("==> It took {} s".format(round(end-begin, 2))) - spark.stop() diff --git a/examples/UDF-Examples/Spark-cuSpatial/src/main/java/com/nvidia/spark/rapids/udf/PointInPolygon.java b/examples/UDF-Examples/Spark-cuSpatial/src/main/java/com/nvidia/spark/rapids/udf/PointInPolygon.java deleted file mode 100644 index 9ae2a4b97..000000000 --- a/examples/UDF-Examples/Spark-cuSpatial/src/main/java/com/nvidia/spark/rapids/udf/PointInPolygon.java +++ /dev/null @@ -1,162 +0,0 @@ -/* - * Copyright (c) 2022, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.nvidia.spark.rapids.udf; - -import ai.rapids.cudf.ColumnVector; -import com.nvidia.spark.RapidsUDF; -import org.apache.spark.SparkFiles; -import org.apache.spark.sql.api.java.UDF2; -import org.apache.spark.sql.internal.SQLConf; - -import java.io.File; -import java.util.List; - -/** - * A Spark Java UDF that computes what the `spatial_join` test does here. - * https://github.com/zhangjianting/cuspatial_benchmark_nyctaxi/blob/main/python/spatial_join.py - */ -public class PointInPolygon implements UDF2>, RapidsUDF { - - private String shapeFile; - private boolean localPathParsed; - - public PointInPolygon() {} - - public PointInPolygon(String shapeFileName) { - this.shapeFile = shapeFileName; - } - - /** Row-by-row implementation that executes on the CPU */ - @Override - public List call(Double x, Double y) { - // not supported yet - throw new UnsupportedOperationException(); - } - - // Share the polygon columns across UDF instances, since each task will create a UDF. - private final static ColumnVector[] polygons = new ColumnVector[4]; - private static int refCount; - - private void ensureShapeFile() { - // Read the config from SQLConf to support runtime updating - String newShapeName = SQLConf.get().getConfString("spark.cuspatial.sql.udf.shapeFileName", null); - boolean validNewName = newShapeName != null && !newShapeName.equals(shapeFile); - // Each task has a different UDF instance so no need to sync when operating object members. - if (!localPathParsed || validNewName) { - if (validNewName) { - // update to the latest - shapeFile = newShapeName; - } - if (shapeFile == null || shapeFile.isEmpty()) { - throw new RuntimeException("Shape file name is missing"); - } - // Get the local path of the downloaded file on each work node. - shapeFile = SparkFiles.get(new File(shapeFile).getName()); - localPathParsed = true; - } - - // load the shape data when needed - synchronized (polygons) { - if (refCount == 0) { - long[] ret = readPolygon(shapeFile); - try { - assert ret.length == polygons.length; - // Table is not applicable here because the columns may have different row numbers. - // So need to cache it as an array. - for (int i = 0; i < polygons.length; i++) { - polygons[i] = new ColumnVector(ret[i]); - } - } catch (Throwable t) { - for (int i = 0; i < polygons.length; i++) { - if (polygons[i] != null) { - polygons[i].close(); - polygons[i] = null; - } else if (ret[i] != 0){ - deleteCudfColumn(ret[i]); - } - } - throw t; - } - } // end of 'if' - // "refCount < 0" can not happen because this method is private. - // the order of increasing/decreasing the reference is managed internally. - refCount ++; - } - } - - private void releaseShapeData() { - synchronized (polygons) { - refCount --; - if (refCount == 0) { - for (int i = 0; i < polygons.length; i++) { - if (polygons[i] != null) { - polygons[i].close(); - polygons[i] = null; - } - } - } - // "refCount < 0" can not happen because this method is private. - // the order of increasing/decreasing the reference is managed internally. - } - } - - /** Columnar implementation that processes data on the GPU */ - @Override - public ColumnVector evaluateColumnar(int numRows, ColumnVector... args) { - if (args.length != 2) { - throw new IllegalArgumentException("Unexpected argument count: " + args.length + - ", expects 2 for (x, y)"); - } - - // Load the native code if it has not been already loaded. This is done here - // rather than in a static code block because the driver may not have the - // required CUDA environment. - UDFNativeLoader.ensureLoaded(); - ensureShapeFile(); - - try { - return new ColumnVector(pointInPolygon(args[0].getNativeView(), args[1].getNativeView(), - polygons[0].getNativeView(), polygons[1].getNativeView(), - polygons[2].getNativeView(), polygons[3].getNativeView())); - } finally { - // Now try to open/close the shape file per call. This can avoid the duplicate readings - // when multiple tasks in the same executors run in parallel. But each batch in the - // same task still executes a new reading. - // A better solution is to try to open/close the shape file per task, then it can also - // avoid duplicate readings per batch in the same task. - // - // All of this is to figure out a proper time to close the columns of the shape data. - releaseShapeData(); - } - } - - private static native void deleteCudfColumn(long cudfColumnHandle); - - /** - * read the polygon shape file as array of 4 columns representing one or more polygons: - * [0]-INT32: beginning index of the first ring in each polygon - * [1]-INT32: beginning index of the first point in each ring - * [2]-FLOAT64: x component of polygon points - * [3]-FLOAT64: y component of polygon points - */ - private static native long[] readPolygon(String shapeFile); - - /** Native implementation that computes on the GPU */ - private static native long pointInPolygon(long xColView, long yColView, - long plyFPosView, long plyRPosView, - long plyXView, long plyYView); -} diff --git a/examples/UDF-Examples/Spark-cuSpatial/src/main/java/com/nvidia/spark/rapids/udf/UDFNativeLoader.java b/examples/UDF-Examples/Spark-cuSpatial/src/main/java/com/nvidia/spark/rapids/udf/UDFNativeLoader.java deleted file mode 100644 index e3c8fa8aa..000000000 --- a/examples/UDF-Examples/Spark-cuSpatial/src/main/java/com/nvidia/spark/rapids/udf/UDFNativeLoader.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright (c) 2022, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.nvidia.spark.rapids.udf; - -import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.net.URL; - -/** Loads the native dependencies for UDFs with a native implementation */ -public class UDFNativeLoader { - private static final ClassLoader loader = UDFNativeLoader.class.getClassLoader(); - private static boolean isLoaded; - - /** Loads native UDF code if necessary */ - public static synchronized void ensureLoaded() { - if (!isLoaded) { - try { - String os = System.getProperty("os.name"); - String arch = System.getProperty("os.arch"); - File path = createFile(os, arch, "spatialudfjni"); - System.load(path.getAbsolutePath()); - isLoaded = true; - } catch (IOException e) { - throw new RuntimeException(e); - } - } - } - - /** Extract the contents of a library resource into a temporary file */ - private static File createFile(String os, String arch, String baseName) throws IOException { - String path = arch + "/" + os + "/" + System.mapLibraryName(baseName); - File loc; - URL resource = loader.getResource(path); - if (resource == null) { - throw new FileNotFoundException("Could not locate native dependency " + path); - } - try (InputStream in = resource.openStream()) { - loc = File.createTempFile(baseName, ".so"); - loc.deleteOnExit(); - try (OutputStream out = new FileOutputStream(loc)) { - byte[] buffer = new byte[1024 * 16]; - int read = 0; - while ((read = in.read(buffer)) >= 0) { - out.write(buffer, 0, read); - } - } - } - return loc; - } -} diff --git a/examples/UDF-Examples/Spark-cuSpatial/src/main/native/CMakeLists.txt b/examples/UDF-Examples/Spark-cuSpatial/src/main/native/CMakeLists.txt deleted file mode 100755 index 4cf897378..000000000 --- a/examples/UDF-Examples/Spark-cuSpatial/src/main/native/CMakeLists.txt +++ /dev/null @@ -1,115 +0,0 @@ -#============================================================================= -# Copyright (c) 2022, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#============================================================================= - -cmake_minimum_required(VERSION 3.20.1 FATAL_ERROR) - -project(SPATIALUDJNI VERSION 23.02.0 LANGUAGES C CXX CUDA) - -################################################################################################### -# - build type ------------------------------------------------------------------------------------ - -# Set a default build type if none was specified -set(DEFAULT_BUILD_TYPE "Release") - -################################################################################################### -# - compiler options ------------------------------------------------------------------------------ - -set(CMAKE_POSITION_INDEPENDENT_CODE ON) -set(CMAKE_CXX_STANDARD 17) -set(CMAKE_CXX_COMPILER $ENV{CXX}) -set(CMAKE_CXX_STANDARD_REQUIRED ON) - -if(CMAKE_COMPILER_IS_GNUCXX) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unknown-pragmas") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=deprecated-declarations") -endif(CMAKE_COMPILER_IS_GNUCXX) - -#################################################################################################### -# - find cuda toolkit ------------------------------------------------------------------------------ -find_package(CUDAToolkit REQUIRED) - -#################################################################################################### -# - find cudf -------------------------------------------------------------------------------------- - -find_path( - CUDF_INCLUDE "cudf" - HINTS "$ENV{CONDA_PREFIX}/include" - "${CONDA_PREFIX}/include" - "$ENV{CUDF_ROOT}/include" - "${CUDF_ROOT}/include" - REQUIRED) - -find_library( - CUDF_LIB "cudf" - HINTS "$ENV{CONDA_PREFIX}/lib" - "${CONDA_PREFIX}/lib" - "$ENV{CUDF_ROOT}/lib" - "${CUDF_ROOT}/lib" - REQUIRED) - -set(CUDACXX_INCLUDE "$ENV{CONDA_PREFIX}/include/rapids/libcudacxx") - -#################################################################################################### -# - find cuspatial -------------------------------------------------------------------------------------- - -find_path( - CUSPATIAL_INCLUDE "cuspatial" - HINTS "$ENV{CONDA_PREFIX}/include" - "${CONDA_PREFIX}/include" - "$ENV{CUSPATIAL_ROOT}/include" - "${CUSPATIAL_ROOT}/include" - REQUIRED) - -find_library( - CUSPATIAL_LIB "cuspatial" - HINTS "$ENV{CONDA_PREFIX}/lib" - "${CONDA_PREFIX}/lib" - "$ENV{CUSPATIAL_ROOT}/lib" - "${CUSPATIAL_ROOT}/lib" - REQUIRED) - -################################################################################################### -# - find JNI ------------------------------------------------------------------------------------- - -find_package(JNI REQUIRED) -if(JNI_FOUND) - message(STATUS "JDK with JNI in ${JNI_INCLUDE_DIRS}") -else() - message(FATAL_ERROR "JDK with JNI not found, please check your environment") -endif(JNI_FOUND) - -################################################################################################### -# - library targets ------------------------------------------------------------------------------- - -add_library(spatialudfjni SHARED "src/PointInPolygonJni.cpp") - -# Override RPATH -set_target_properties(spatialudfjni PROPERTIES BUILD_RPATH "\$ORIGIN") - -# include directories -target_include_directories( - spatialudfjni - PUBLIC "${JNI_INCLUDE_DIRS}" - "${CUDF_INCLUDE}" - "${CUDACXX_INCLUDE}" - "${CUDAToolkit_INCLUDE_DIRS}" - "${CUSPATIAL_INCLUDE}") - -################################################################################################### -# - link libraries -------------------------------------------------------------------------------- - -target_link_libraries(spatialudfjni ${CUDF_LIB} ${CUSPATIAL_LIB}) diff --git a/examples/UDF-Examples/Spark-cuSpatial/src/main/native/src/PointInPolygonJni.cpp b/examples/UDF-Examples/Spark-cuSpatial/src/main/native/src/PointInPolygonJni.cpp deleted file mode 100644 index ecbc0b6b2..000000000 --- a/examples/UDF-Examples/Spark-cuSpatial/src/main/native/src/PointInPolygonJni.cpp +++ /dev/null @@ -1,345 +0,0 @@ -/* - * Copyright (c) 2022, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#include -#include -#include -#include - -namespace { - -constexpr char const* RUNTIME_ERROR_CLASS = "java/lang/RuntimeException"; -constexpr char const* ILLEGAL_ARG_CLASS = "java/lang/IllegalArgumentException"; - -/** - * @brief Throw a Java exception - * - * @param env The Java environment - * @param class_name The fully qualified Java class name of the exception - * @param msg The message string to associate with the exception - */ -void throw_java_exception(JNIEnv* env, char const* class_name, char const* msg) { - jclass ex_class = env->FindClass(class_name); - if (ex_class != NULL) { - env->ThrowNew(ex_class, msg); - } -} - -/** - * @brief check if an java exceptions have been thrown and if so throw a C++ - * exception so the flow control stop processing. - */ -inline void check_java_exception(JNIEnv *const env) { - if (env->ExceptionCheck()) { - // Not going to try to get the message out of the Exception, too complex and - // might fail. - throw std::runtime_error("JNI Exception..."); - } -} - -/** @brief RAII for jstring to be sure it is handled correctly. */ -class native_jstring { -private: - JNIEnv *env; - jstring orig; - mutable const char *cstr; - mutable size_t cstr_length; - - void init_cstr() const { - if (orig != NULL && cstr == NULL) { - cstr_length = env->GetStringUTFLength(orig); - cstr = env->GetStringUTFChars(orig, 0); - check_java_exception(env); - } - } - -public: - native_jstring(native_jstring const &) = delete; - native_jstring &operator=(native_jstring const &) = delete; - - native_jstring(native_jstring &&other) noexcept - : env(other.env), orig(other.orig), cstr(other.cstr), cstr_length(other.cstr_length) { - other.cstr = NULL; - } - - native_jstring(JNIEnv *const env, jstring orig) - : env(env), orig(orig), cstr(NULL), cstr_length(0) {} - - native_jstring &operator=(native_jstring const &&other) { - if (orig != NULL && cstr != NULL) { - env->ReleaseStringUTFChars(orig, cstr); - } - this->env = other.env; - this->orig = other.orig; - this->cstr = other.cstr; - this->cstr_length = other.cstr_length; - other.cstr = NULL; - return *this; - } - - bool is_null() const noexcept { return orig == NULL; } - - const char *get() const { - init_cstr(); - return cstr; - } - - ~native_jstring() { - if (orig != NULL && cstr != NULL) { - env->ReleaseStringUTFChars(orig, cstr); - } - } -}; - -/** - * @brief a column is valid only when it has at least one valid row. - */ -inline bool is_invalid_column(cudf::column_view const& col) { - return col.null_count() == col.size(); -} - -/** - * @brief run the reduction 'agg' on the input column 'col'. The input column should be - * double type, and have at least one valid row. Otherwise, the behavior is undefined. - */ -inline double reduce_as_double(cudf::column_view const& col, - cudf::reduce_aggregation const& agg) { - auto s = cudf::reduce(col, agg, col.type()); - // s is always valid - auto p_num_scalar = reinterpret_cast*>(s.get()); - return p_num_scalar->value(); -} - -/** - * @brief convert the cuspatial result for the 'point_in_polygon' test, - * to match the simple UDF output requirement. - * - * (It is called only once so it can be inline.) - */ -inline jlong convert_point_in_polygon_result(std::unique_ptr pt_indices, - std::unique_ptr ply_indices, - cudf::column_view const& raw_xv, - cudf::column_view const& raw_yv) { - // Sort result by (point_offset, polygon_offset) pair, to make the output deterministic. - // The sorted polygon index column can be used for the output LIST child column. - auto sorted_table_ptr = cudf::sort(cudf::table_view({ - pt_indices->get_column(0), - *ply_indices})); - // release the GPU resources once they are no longer needed. - pt_indices.reset(); - ply_indices.reset(); - - // Compute the offsets of the output by running the lower_bound with a sequence starting from 0. - auto raw_pt_indices = cudf::sequence(raw_xv.size() + 1, cudf::numeric_scalar(0)); - auto offset_col = cudf::lower_bound( - cudf::table_view({sorted_table_ptr->get_column(0)}), - cudf::table_view({*raw_pt_indices}), - {}, {}); - raw_pt_indices.reset(); - - // Cast the child column from UINT32 to INT32 (int32 should be enough) for output - auto child_col = cudf::cast(sorted_table_ptr->get_column(1), cudf::data_type(cudf::type_id::INT32)); - // release the GPU resources once they are no longer needed. - sorted_table_ptr.reset(); - - // Compute the validity mask of the output by 'bitwise-AND'ing the two input point coordinate - // column validity masks together. - auto [new_mask, new_null_count] = cudf::bitmask_and(cudf::table_view({raw_xv, raw_yv})); - - // Now all elements are ready, create the final list column and return it. - auto output_list = cudf::make_lists_column( - raw_xv.size(), - std::move(offset_col), - std::move(child_col), - new_null_count, - std::move(new_mask)); - return reinterpret_cast(output_list.release()); -} - -static jlongArray convert_cols_for_return(JNIEnv *env, - std::vector>& ret) { - int cols_num = ret.size(); - jlongArray j_out_handles = env->NewLongArray(cols_num); - jlong* out_handles = env->GetLongArrayElements(j_out_handles, NULL); - check_java_exception(env); - if (j_out_handles == nullptr || out_handles == nullptr) { - throw std::bad_alloc(); - } - for (int i = 0; i < cols_num; i++) { - out_handles[i] = reinterpret_cast(ret[i].release()); - } - env->ReleaseLongArrayElements(j_out_handles, out_handles, 0); - return j_out_handles; -} - -} // anonymous namespace - -extern "C" { - -JNIEXPORT void JNICALL -Java_com_nvidia_spark_rapids_udf_PointInPolygon_deleteCudfColumn(JNIEnv* env, jclass, - jlong j_handle) { - delete reinterpret_cast(j_handle); -} - -JNIEXPORT jlongArray JNICALL -Java_com_nvidia_spark_rapids_udf_PointInPolygon_readPolygon(JNIEnv* env, jclass, - jstring j_shape_file) { - // turn the Java string to the native string. - // The file is always valid, which is ensured by Java. - native_jstring shape_file(env, j_shape_file); - auto poly_cols = cuspatial::read_polygon_shapefile(shape_file.get()); - return convert_cols_for_return(env, poly_cols); -} - -/** - * @brief The native implementation of PointInPolygon.pointInPolygon. - * - * @param env The Java environment - * @param j_x_view The address of the cudf column view of the x column - * @param j_y_view The address of the cudf column view of the y column - * @param j_ply_fpos The column address of beginning index of the first ring in each polygon - * @param j_ply_rpos The column address of beginning index of the first point in each ring - * @param j_ply_x The column address of x component of polygon points - * @param j_ply_y The column address of y component of polygon points - * @return The address of the cudf column containing the results - */ -JNIEXPORT jlong JNICALL -Java_com_nvidia_spark_rapids_udf_PointInPolygon_pointInPolygon(JNIEnv* env, jclass, - jlong j_x_view, jlong j_y_view, - jlong j_ply_fpos, jlong j_ply_rpos, - jlong j_ply_x, jlong j_ply_y) { - // Use a try block to translate C++ exceptions into Java exceptions to avoid - // crashing the JVM if a C++ exception occurs. - try { - // turn the addresses into column_view pointers - auto raw_xv = reinterpret_cast(j_x_view); - auto raw_yv = reinterpret_cast(j_y_view); - if (raw_xv->type().id() != raw_yv->type().id()) { - throw_java_exception(env, ILLEGAL_ARG_CLASS, "x and y have different types"); - return 0; - } - - // The python test casts the data to numpy.float32, but here cudf FLOAT32 does not work. - // It complains the error - // " quadtree_point_in_polygon.cu:275: points and polygons must have the same data type" - // - // so convert int32 to cudf FLOAT64 and it works. - auto xv = cudf::cast(*raw_xv, cudf::data_type(cudf::type_id::FLOAT64)); - auto yv = cudf::cast(*raw_yv, cudf::data_type(cudf::type_id::FLOAT64)); - - // check shape data - auto ply_fpos = reinterpret_cast(j_ply_fpos); - auto ply_rpos = reinterpret_cast(j_ply_rpos); - auto ply_x = reinterpret_cast(j_ply_x); - auto ply_y = reinterpret_cast(j_ply_y); - - if (is_invalid_column(*ply_x) || is_invalid_column(*ply_y)) { - // No polygon data, then return a list column of all nulls. - // The offsets are all 0, and the child is an empty column. - auto offset_col = cudf::make_column_from_scalar(cudf::numeric_scalar(0), - xv->size() + 1); - auto nulls_list = cudf::make_lists_column( - xv->size(), - std::move(offset_col), - cudf::make_empty_column(cudf::type_id::INT32), - xv->size(), - cudf::create_null_mask(xv->size(), cudf::mask_state::ALL_NULL)); - return reinterpret_cast(nulls_list.release()); - } - - auto min_agg = cudf::make_min_aggregation(); - auto max_agg = cudf::make_max_aggregation(); - auto x_min = reduce_as_double(*ply_x, *min_agg); - auto x_max = reduce_as_double(*ply_x, *max_agg); - auto y_min = reduce_as_double(*ply_y, *min_agg); - auto y_max = reduce_as_double(*ply_y, *max_agg); - - // 2) quadtree construction - cudf::size_type min_size = 512; - int8_t num_levels = 15; - double scale = std::max(std::abs(x_max - x_min), std::abs(y_max - y_min)) / ((1 << num_levels) - 2); - - auto [point_indices, quadtree] = cuspatial::quadtree_on_points( - *xv, *yv, x_min, x_max, y_min, y_max, scale, num_levels, min_size); - - // 3) run the computation - auto poly_bboxes = cuspatial::polygon_bounding_boxes(*ply_fpos, *ply_rpos, *ply_x, *ply_y); - - auto poly_quadrant_pairs = cuspatial::join_quadtree_and_bounding_boxes( - *quadtree, *poly_bboxes, x_min, x_max, y_min, y_max, scale, num_levels); - // release the GPU resources once they are no longer needed. - poly_bboxes.reset(); - - auto point_in_polygon_pairs = cuspatial::quadtree_point_in_polygon( - *poly_quadrant_pairs, *quadtree, *point_indices, - *xv, *yv, - *ply_fpos, *ply_rpos, *ply_x, *ply_y); - // release the GPU resources once they are no longer needed. - poly_quadrant_pairs.reset(); - quadtree.reset(); - - // The result table has two columns, where each row represents a - // (polygon_offset, point_offset) pair: - // - polygon_offset - UINT32 column of polygon indices - // - point_offset - UINT32 column of point indices - auto ply_pt_offset = point_in_polygon_pairs->release(); - - // 4) convert the `point_offset` to indices of the input points - auto input_pt_indices = cudf::gather( - cudf::table_view({*point_indices}), - *(ply_pt_offset[1])); - // release the GPU resources once they are no longer needed. - point_indices.reset(); - ply_pt_offset[1].reset(); - - // 4) convert the cuspatail result and return. - return convert_point_in_polygon_result( - std::move(input_pt_indices), std::move(ply_pt_offset[0]), *xv, *yv); - - } catch (std::bad_alloc const& e) { - auto msg = std::string("Unable to allocate native memory: ") + - (e.what() == nullptr ? "" : e.what()); - throw_java_exception(env, RUNTIME_ERROR_CLASS, msg.c_str()); - - } catch (std::invalid_argument const& e) { - auto msg = e.what() == nullptr ? "" : e.what(); - throw_java_exception(env, ILLEGAL_ARG_CLASS, msg); - - } catch (std::exception const& e) { - auto msg = e.what() == nullptr ? "" : e.what(); - throw_java_exception(env, RUNTIME_ERROR_CLASS, msg); - } - return 0; -} - -} diff --git a/examples/UDF-Examples/Spark-cuSpatial/tools/to_parquet.py b/examples/UDF-Examples/Spark-cuSpatial/tools/to_parquet.py deleted file mode 100644 index b6219fe87..000000000 --- a/examples/UDF-Examples/Spark-cuSpatial/tools/to_parquet.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import cudf -import numpy as np -import cupy -import sys - - -def read_points(path): - print('reading points file:', path) - points = np.fromfile(path, dtype=np.int32) - points = cupy.asarray(points) - points = points.reshape((len(points)// 4, 4)) - points = cudf.DataFrame(points) - points_df = cudf.DataFrame() - points_df['x'] = points[0] - points_df['y'] = points[1] - return points_df - -if __name__ == '__main__': - if len(sys.argv) < 3: - raise Exception("Usage: to_parquet .") - inputPath = sys.argv[1] - outputPath = sys.argv[2] - - points_df = read_points(inputPath) - points_df.to_parquet(outputPath) - diff --git a/examples/XGBoost-Examples/agaricus/notebooks/python/agaricus-gpu.ipynb b/examples/XGBoost-Examples/agaricus/notebooks/python/agaricus-gpu.ipynb index 74278239a..7be0e60f4 100644 --- a/examples/XGBoost-Examples/agaricus/notebooks/python/agaricus-gpu.ipynb +++ b/examples/XGBoost-Examples/agaricus/notebooks/python/agaricus-gpu.ipynb @@ -73,7 +73,7 @@ "Setting default log level to \"WARN\".\n", "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n", "2022-11-30 06:57:40,550 WARN resource.ResourceUtils: The configuration of cores (exec = 2 task = 1, runnable tasks = 2) will result in wasted resources due to resource gpu limiting the number of runnable tasks per executor to: 1. Please adjust your configuration.\n", - "2022-11-30 06:57:54,195 WARN rapids.RapidsPluginUtils: RAPIDS Accelerator 24.06.0 using cudf 24.06.0.\n", + "2022-11-30 06:57:54,195 WARN rapids.RapidsPluginUtils: RAPIDS Accelerator 24.08.1 using cudf 24.08.1.\n", "2022-11-30 06:57:54,210 WARN rapids.RapidsPluginUtils: spark.rapids.sql.multiThreadedRead.numThreads is set to 20.\n", "2022-11-30 06:57:54,214 WARN rapids.RapidsPluginUtils: RAPIDS Accelerator is enabled, to disable GPU support set `spark.rapids.sql.enabled` to false.\n", "2022-11-30 06:57:54,214 WARN rapids.RapidsPluginUtils: spark.rapids.sql.explain is set to `NOT_ON_GPU`. Set it to 'NONE' to suppress the diagnostics logging about the query placement on the GPU.\n", diff --git a/examples/XGBoost-Examples/mortgage/notebooks/python/MortgageETL.ipynb b/examples/XGBoost-Examples/mortgage/notebooks/python/MortgageETL.ipynb index 9622efb1f..02875a23a 100644 --- a/examples/XGBoost-Examples/mortgage/notebooks/python/MortgageETL.ipynb +++ b/examples/XGBoost-Examples/mortgage/notebooks/python/MortgageETL.ipynb @@ -6,10 +6,10 @@ "source": [ "## Prerequirement\n", "### 1. Download data\n", - "Dataset is derived from Fannie Mae’s [Single-Family Loan Performance Data](http://www.fanniemae.com/portal/funding-the-market/data/loan-performance-data.html) with all rights reserved by Fannie Mae. Refer to these [instructions](https://github.com/NVIDIA/spark-rapids-examples/blob/branch-24.06/docs/get-started/xgboost-examples/dataset/mortgage.md) to download the dataset.\n", + "Dataset is derived from Fannie Mae’s [Single-Family Loan Performance Data](http://www.fanniemae.com/portal/funding-the-market/data/loan-performance-data.html) with all rights reserved by Fannie Mae. Refer to these [instructions](https://github.com/NVIDIA/spark-rapids-examples/blob/branch-24.08/docs/get-started/xgboost-examples/dataset/mortgage.md) to download the dataset.\n", "\n", "### 2. Download needed jars\n", - "* [rapids-4-spark_2.12-24.06.0.jar](https://repo1.maven.org/maven2/com/nvidia/rapids-4-spark_2.12/24.06.0/rapids-4-spark_2.12-24.06.0.jar)\n", + "* [rapids-4-spark_2.12-24.08.1.jar](https://repo1.maven.org/maven2/com/nvidia/rapids-4-spark_2.12/24.08.1/rapids-4-spark_2.12-24.08.1.jar)\n", "\n", "\n", "### 3. Start Spark Standalone\n", @@ -17,7 +17,7 @@ "\n", "### 4. Add ENV\n", "```\n", - "$ export SPARK_JARS=rapids-4-spark_2.12-24.06.0.jar\n", + "$ export SPARK_JARS=rapids-4-spark_2.12-24.08.1.jar\n", "$ export PYSPARK_DRIVER_PYTHON=jupyter \n", "$ export PYSPARK_DRIVER_PYTHON_OPTS=notebook\n", "```\n", diff --git a/examples/XGBoost-Examples/mortgage/notebooks/python/cv-mortgage-gpu.ipynb b/examples/XGBoost-Examples/mortgage/notebooks/python/cv-mortgage-gpu.ipynb index cdc4d1ec5..fd1526467 100644 --- a/examples/XGBoost-Examples/mortgage/notebooks/python/cv-mortgage-gpu.ipynb +++ b/examples/XGBoost-Examples/mortgage/notebooks/python/cv-mortgage-gpu.ipynb @@ -63,7 +63,7 @@ "Setting default log level to \"WARN\".\n", "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n", "2022-11-25 09:34:43,952 WARN resource.ResourceUtils: The configuration of cores (exec = 4 task = 1, runnable tasks = 4) will result in wasted resources due to resource gpu limiting the number of runnable tasks per executor to: 1. Please adjust your configuration.\n", - "2022-11-25 09:34:58,155 WARN rapids.RapidsPluginUtils: RAPIDS Accelerator 24.06.0 using cudf 24.06.0.\n", + "2022-11-25 09:34:58,155 WARN rapids.RapidsPluginUtils: RAPIDS Accelerator 24.08.1 using cudf 24.08.1.\n", "2022-11-25 09:34:58,171 WARN rapids.RapidsPluginUtils: spark.rapids.sql.multiThreadedRead.numThreads is set to 20.\n", "2022-11-25 09:34:58,175 WARN rapids.RapidsPluginUtils: RAPIDS Accelerator is enabled, to disable GPU support set `spark.rapids.sql.enabled` to false.\n", "2022-11-25 09:34:58,175 WARN rapids.RapidsPluginUtils: spark.rapids.sql.explain is set to `NOT_ON_GPU`. Set it to 'NONE' to suppress the diagnostics logging about the query placement on the GPU.\n" diff --git a/examples/XGBoost-Examples/mortgage/notebooks/python/mortgage-gpu.ipynb b/examples/XGBoost-Examples/mortgage/notebooks/python/mortgage-gpu.ipynb index 590a68e37..f3f8ab047 100644 --- a/examples/XGBoost-Examples/mortgage/notebooks/python/mortgage-gpu.ipynb +++ b/examples/XGBoost-Examples/mortgage/notebooks/python/mortgage-gpu.ipynb @@ -84,7 +84,7 @@ "22/11/24 06:14:06 INFO org.apache.spark.SparkEnv: Registering BlockManagerMaster\n", "22/11/24 06:14:06 INFO org.apache.spark.SparkEnv: Registering BlockManagerMasterHeartbeat\n", "22/11/24 06:14:06 INFO org.apache.spark.SparkEnv: Registering OutputCommitCoordinator\n", - "22/11/24 06:14:07 WARN com.nvidia.spark.rapids.RapidsPluginUtils: RAPIDS Accelerator 24.06.0 using cudf 24.06.0.\n", + "22/11/24 06:14:07 WARN com.nvidia.spark.rapids.RapidsPluginUtils: RAPIDS Accelerator 24.08.1 using cudf 24.08.1.\n", "22/11/24 06:14:07 WARN com.nvidia.spark.rapids.RapidsPluginUtils: spark.rapids.sql.multiThreadedRead.numThreads is set to 20.\n", "22/11/24 06:14:07 WARN com.nvidia.spark.rapids.RapidsPluginUtils: RAPIDS Accelerator is enabled, to disable GPU support set `spark.rapids.sql.enabled` to false.\n", "22/11/24 06:14:07 WARN com.nvidia.spark.rapids.RapidsPluginUtils: spark.rapids.sql.explain is set to `NOT_ON_GPU`. Set it to 'NONE' to suppress the diagnostics logging about the query placement on the GPU.\n" diff --git a/examples/XGBoost-Examples/mortgage/notebooks/scala/mortgage-ETL.ipynb b/examples/XGBoost-Examples/mortgage/notebooks/scala/mortgage-ETL.ipynb index 43bad4fdb..a2725240c 100644 --- a/examples/XGBoost-Examples/mortgage/notebooks/scala/mortgage-ETL.ipynb +++ b/examples/XGBoost-Examples/mortgage/notebooks/scala/mortgage-ETL.ipynb @@ -20,14 +20,14 @@ "Refer to these [instructions](https://github.com/NVIDIA/spark-rapids-examples/blob/branch-23.12/docs/get-started/xgboost-examples/dataset/mortgage.md) to download the dataset.\n", "\n", "### 2. Download needed jars\n", - "* [rapids-4-spark_2.12-24.06.0.jar](https://repo1.maven.org/maven2/com/nvidia/rapids-4-spark_2.12/24.06.0/rapids-4-spark_2.12-24.06.0.jar)\n", + "* [rapids-4-spark_2.12-24.08.1.jar](https://repo1.maven.org/maven2/com/nvidia/rapids-4-spark_2.12/24.08.1/rapids-4-spark_2.12-24.08.1.jar)\n", "\n", "### 3. Start Spark Standalone\n", "Before Running the script, please setup Spark standalone mode\n", "\n", "### 4. Add ENV\n", "```\n", - "$ export SPARK_JARS=rapids-4-spark_2.12-24.06.0.jar\n", + "$ export SPARK_JARS=rapids-4-spark_2.12-24.08.1.jar\n", "\n", "```\n", "\n", diff --git a/examples/XGBoost-Examples/taxi/notebooks/python/cv-taxi-gpu.ipynb b/examples/XGBoost-Examples/taxi/notebooks/python/cv-taxi-gpu.ipynb index 1ca71c7e8..35dc78f29 100644 --- a/examples/XGBoost-Examples/taxi/notebooks/python/cv-taxi-gpu.ipynb +++ b/examples/XGBoost-Examples/taxi/notebooks/python/cv-taxi-gpu.ipynb @@ -62,7 +62,7 @@ "Setting default log level to \"WARN\".\n", "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n", "2022-11-30 08:02:10,103 WARN resource.ResourceUtils: The configuration of cores (exec = 2 task = 1, runnable tasks = 2) will result in wasted resources due to resource gpu limiting the number of runnable tasks per executor to: 1. Please adjust your configuration.\n", - "2022-11-30 08:02:23,737 WARN rapids.RapidsPluginUtils: RAPIDS Accelerator 24.06.0 using cudf 24.06.0.\n", + "2022-11-30 08:02:23,737 WARN rapids.RapidsPluginUtils: RAPIDS Accelerator 24.08.1 using cudf 24.08.1.\n", "2022-11-30 08:02:23,752 WARN rapids.RapidsPluginUtils: spark.rapids.sql.multiThreadedRead.numThreads is set to 20.\n", "2022-11-30 08:02:23,756 WARN rapids.RapidsPluginUtils: RAPIDS Accelerator is enabled, to disable GPU support set `spark.rapids.sql.enabled` to false.\n", "2022-11-30 08:02:23,757 WARN rapids.RapidsPluginUtils: spark.rapids.sql.explain is set to `NOT_ON_GPU`. Set it to 'NONE' to suppress the diagnostics logging about the query placement on the GPU.\n", diff --git a/examples/XGBoost-Examples/taxi/notebooks/python/taxi-ETL.ipynb b/examples/XGBoost-Examples/taxi/notebooks/python/taxi-ETL.ipynb index 911d6e8be..b00765d46 100644 --- a/examples/XGBoost-Examples/taxi/notebooks/python/taxi-ETL.ipynb +++ b/examples/XGBoost-Examples/taxi/notebooks/python/taxi-ETL.ipynb @@ -19,14 +19,14 @@ "All data could be found at https://www1.nyc.gov/site/tlc/about/tlc-trip-record-data.page\n", "\n", "### 2. Download needed jars\n", - "* [rapids-4-spark_2.12-24.06.0.jar](https://repo1.maven.org/maven2/com/nvidia/rapids-4-spark_2.12/24.06.0/rapids-4-spark_2.12-24.06.0.jar)\n", + "* [rapids-4-spark_2.12-24.08.1.jar](https://repo1.maven.org/maven2/com/nvidia/rapids-4-spark_2.12/24.08.1/rapids-4-spark_2.12-24.08.1.jar)\n", "\n", "### 3. Start Spark Standalone\n", "Before running the script, please setup Spark standalone mode\n", "\n", "### 4. Add ENV\n", "```\n", - "$ export SPARK_JARS=rapids-4-spark_2.12-24.06.0.jar\n", + "$ export SPARK_JARS=rapids-4-spark_2.12-24.08.1.jar\n", "$ export PYSPARK_DRIVER_PYTHON=jupyter \n", "$ export PYSPARK_DRIVER_PYTHON_OPTS=notebook\n", "```\n", diff --git a/examples/XGBoost-Examples/taxi/notebooks/python/taxi-gpu.ipynb b/examples/XGBoost-Examples/taxi/notebooks/python/taxi-gpu.ipynb index 2877030cc..078d3ad5c 100644 --- a/examples/XGBoost-Examples/taxi/notebooks/python/taxi-gpu.ipynb +++ b/examples/XGBoost-Examples/taxi/notebooks/python/taxi-gpu.ipynb @@ -73,7 +73,7 @@ "Setting default log level to \"WARN\".\n", "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n", "2022-11-30 07:51:19,480 WARN resource.ResourceUtils: The configuration of cores (exec = 2 task = 1, runnable tasks = 2) will result in wasted resources due to resource gpu limiting the number of runnable tasks per executor to: 1. Please adjust your configuration.\n", - "2022-11-30 07:51:33,277 WARN rapids.RapidsPluginUtils: RAPIDS Accelerator 24.06.0 using cudf 24.06.0.\n", + "2022-11-30 07:51:33,277 WARN rapids.RapidsPluginUtils: RAPIDS Accelerator 24.08.1 using cudf 24.08.1.\n", "2022-11-30 07:51:33,292 WARN rapids.RapidsPluginUtils: spark.rapids.sql.multiThreadedRead.numThreads is set to 20.\n", "2022-11-30 07:51:33,295 WARN rapids.RapidsPluginUtils: RAPIDS Accelerator is enabled, to disable GPU support set `spark.rapids.sql.enabled` to false.\n", "2022-11-30 07:51:33,295 WARN rapids.RapidsPluginUtils: spark.rapids.sql.explain is set to `NOT_ON_GPU`. Set it to 'NONE' to suppress the diagnostics logging about the query placement on the GPU.\n", diff --git a/examples/XGBoost-Examples/taxi/notebooks/scala/taxi-ETL.ipynb b/examples/XGBoost-Examples/taxi/notebooks/scala/taxi-ETL.ipynb index 7f96df4bb..979d2f9a9 100644 --- a/examples/XGBoost-Examples/taxi/notebooks/scala/taxi-ETL.ipynb +++ b/examples/XGBoost-Examples/taxi/notebooks/scala/taxi-ETL.ipynb @@ -19,14 +19,14 @@ "All data could be found at https://www1.nyc.gov/site/tlc/about/tlc-trip-record-data.page\n", "\n", "### 2. Download needed jar\n", - "* [rapids-4-spark_2.12-24.06.0.jar](https://repo1.maven.org/maven2/com/nvidia/rapids-4-spark_2.12/24.06.0/rapids-4-spark_2.12-24.06.0.jar)\n", + "* [rapids-4-spark_2.12-24.08.1.jar](https://repo1.maven.org/maven2/com/nvidia/rapids-4-spark_2.12/24.08.1/rapids-4-spark_2.12-24.08.1.jar)\n", "\n", "### 3. Start Spark Standalone\n", "Before running the script, please setup Spark standalone mode\n", "\n", "### 4. Add ENV\n", "```\n", - "$ export SPARK_JARS=rapids-4-spark_2.12-24.06.0.jar\n", + "$ export SPARK_JARS=rapids-4-spark_2.12-24.08.1.jar\n", "\n", "```\n", "\n", diff --git a/tools/databricks/README.md b/tools/databricks/README.md index 26ca07e56..7691e2508 100644 --- a/tools/databricks/README.md +++ b/tools/databricks/README.md @@ -12,8 +12,12 @@ cluster. Once the notebook is activated, you can enter in the log path location top of the notebook. After that, select *Run all* to execute the tools for the specific logs in the log path. ## Limitations -1. Currently only local or DBFS eventlog paths are supported. -2. DBFS path must use the File API Format. -3. Example: `/dbfs/`. +1. Currently local, S3 or DBFS event log paths are supported. +1. S3 path is only supported on Databricks AWS using [instance profiles](https://docs.databricks.com/en/connect/storage/tutorial-s3-instance-profile.html). +1. Eventlog path must follow the formats `/dbfs/path/to/eventlog` or `dbfs:/path/to/eventlog` for logs stored in DBFS. +1. Use wildcards for nested lookup of eventlogs. + - For example: `/dbfs/path/to/clusterlogs/*/*` +1. Multiple event logs must be comma-separated. + - For example: `/dbfs/path/to/eventlog1,/dbfs/path/to/eventlog2` -**Latest Tools Version Supported** 24.06.0 \ No newline at end of file +**Latest Tools Version Supported** 24.08.0 \ No newline at end of file diff --git a/tools/databricks/[RAPIDS Accelerator for Apache Spark] Profiling Tool Notebook Template.ipynb b/tools/databricks/[RAPIDS Accelerator for Apache Spark] Profiling Tool Notebook Template.ipynb index 3cf5bd9ea..503b18ffb 100644 --- a/tools/databricks/[RAPIDS Accelerator for Apache Spark] Profiling Tool Notebook Template.ipynb +++ b/tools/databricks/[RAPIDS Accelerator for Apache Spark] Profiling Tool Notebook Template.ipynb @@ -20,9 +20,13 @@ "To run the profiling tool, enter the log path that represents the DBFS location of your Spark GPU event logs. Then, select \"Run all\" to execute the notebook. Once the notebook completes, various output tables will appear below. For more options on running the profiling tool, please refer to the [Profiling Tool User Guide](https://docs.nvidia.com/spark-rapids/user-guide/latest/profiling/quickstart.html#running-the-tool).\n", "\n", "### Note\n", - "- Currently, only local or DBFS event log paths are supported.\n", - "- The DBFS path must use the File API format. Example: `/dbfs/`.\n", - "- Multiple event logs must be comma-separated.\n", + "- Currently, local, S3 or DBFS event log paths are supported.\n", + "- S3 path is only supported on Databricks AWS using [instance profiles](https://docs.databricks.com/en/connect/storage/tutorial-s3-instance-profile.html).\n", + "- Eventlog path must follow the formats `/dbfs/path/to/eventlog` or `dbfs:/path/to/eventlog` for logs stored in DBFS.\n", + "- Use wildcards for nested lookup of eventlogs. \n", + " - For example: `/dbfs/path/to/clusterlogs/*/*`\n", + "- Multiple event logs must be comma-separated. \n", + " - For example: `/dbfs/path/to/eventlog1,/dbfs/path/to/eventlog2`\n", "\n", "### Per-Job Profile\n", "\n", @@ -49,7 +53,7 @@ }, "outputs": [], "source": [ - "TOOLS_VER = \"24.06.0\"\n", + "TOOLS_VER = \"24.08.0\"\n", "print(f\"Using Tools Version: {TOOLS_VER}\")" ] }, @@ -96,12 +100,23 @@ "import os\n", "import pandas as pd\n", "\n", - "# Initialize variables from widgets\n", - "dbutils.widgets.dropdown(\"Cloud Provider\", \"aws\", [\"aws\", \"azure\"])\n", - "CSP=dbutils.widgets.get(\"Cloud Provider\")\n", "\n", + "def convert_dbfs_path(path):\n", + " return path.replace(\"dbfs:/\", \"/dbfs/\")\n", + " \n", + "# Detect cloud provider from cluster usage tags\n", + "valid_csps = [\"aws\", \"azure\"]\n", + "CSP=spark.conf.get(\"spark.databricks.clusterUsageTags.cloudProvider\", \"\").lower()\n", + "if CSP not in valid_csps:\n", + " print(f\"ERROR: Cannot detect cloud provider from cluster usage tags. Using '{valid_csps[0]}' as default. \")\n", + " CSP = valid_csps[0]\n", + "else:\n", + " print(f\"Detected Cloud Provider from Spark Configs: '{CSP}'\")\n", + "\n", + "# Initialize variables from widgets\n", "dbutils.widgets.text(\"Eventlog Path\", \"/dbfs/user1/profiling_logs\")\n", "EVENTLOG_PATH=dbutils.widgets.get(\"Eventlog Path\")\n", + "EVENTLOG_PATH=convert_dbfs_path(EVENTLOG_PATH)\n", "\n", "dbutils.widgets.text(\"Output Path\", \"/tmp\")\n", "OUTPUT_PATH=dbutils.widgets.get(\"Output Path\")\n", @@ -137,7 +152,7 @@ "outputs": [], "source": [ "%sh\n", - "spark_rapids profiling --platform databricks-$CSP --eventlogs $EVENTLOG_PATH -o $OUTPUT_PATH > $CONSOLE_OUTPUT_PATH 2> $CONSOLE_ERROR_PATH" + "spark_rapids profiling --platform databricks-$CSP --eventlogs \"$EVENTLOG_PATH\" -o \"$OUTPUT_PATH\" --verbose > \"$CONSOLE_OUTPUT_PATH\" 2> \"$CONSOLE_ERROR_PATH\"" ] }, { @@ -156,9 +171,7 @@ }, "source": [ "## Console Output\n", - "Console output shows the recommended configurations for each app\n", - "\n", - "**Note**: Use the `--verbose` flag in the command above for more detailed output.\n" + "Console output shows the recommended configurations for each app\n" ] }, { @@ -197,7 +210,7 @@ "inputWidgets": {}, "nuid": "f3c68b28-fc62-40ae-8528-799f3fc7507e", "showTitle": true, - "title": "Show Error Log" + "title": "Show Logs" }, "jupyter": { "source_hidden": true @@ -451,7 +464,7 @@ "stack": true }, "nuid": "91c1bfb2-695a-4e5c-8a25-848a433108dc", - "origId": 825198511668411, + "origId": 2173122769183713, "title": "Executive View", "version": "DashboardViewV1", "width": 1600 @@ -465,7 +478,7 @@ "stack": true }, "nuid": "62243296-4562-4f06-90ac-d7a609f19c16", - "origId": 825198511668412, + "origId": 2173122769183714, "title": "App View", "version": "DashboardViewV1", "width": 1920 @@ -475,51 +488,27 @@ "language": "python", "notebookMetadata": { "mostRecentlyExecutedCommandWithImplicitDF": { - "commandId": 825198511668406, + "commandId": 2173122769183692, "dataframes": [ "_sqldf" ] }, "pythonIndentUnit": 2, "widgetLayout": [ - { - "breakBefore": false, - "name": "Cloud Provider", - "width": 183 - }, { "breakBefore": false, "name": "Eventlog Path", - "width": 728 + "width": 778 }, { "breakBefore": false, "name": "Output Path", - "width": 232 + "width": 302 } ] }, "notebookName": "[RAPIDS Accelerator for Apache Spark] Profiling Tool Notebook Template", "widgets": { - "Cloud Provider": { - "currentValue": "aws", - "nuid": "8dddcaf7-104e-4247-b811-ff7a133b28d4", - "typedWidgetInfo": null, - "widgetInfo": { - "widgetType": "dropdown", - "defaultValue": "aws", - "label": null, - "name": "Cloud Provider", - "options": { - "widgetType": "dropdown", - "autoCreated": null, - "choices": [ - "aws", - "azure" - ] - } - } - }, "Eventlog Path": { "currentValue": "/dbfs/user1/profiling_logs", "nuid": "1272501d-5ad9-42be-ab62-35768b2fc384", diff --git a/tools/databricks/[RAPIDS Accelerator for Apache Spark] Qualification Tool Notebook Template.ipynb b/tools/databricks/[RAPIDS Accelerator for Apache Spark] Qualification Tool Notebook Template.ipynb index 6b6863ae2..898a4846f 100644 --- a/tools/databricks/[RAPIDS Accelerator for Apache Spark] Qualification Tool Notebook Template.ipynb +++ b/tools/databricks/[RAPIDS Accelerator for Apache Spark] Qualification Tool Notebook Template.ipynb @@ -20,9 +20,13 @@ "To run the qualification tool, enter the log path that represents the DBFS location of your Spark GPU event logs. Then, select \"Run all\" to execute the notebook. Once the notebook completes, various output tables will appear below. For more options on running the profiling tool, please refer to the [Qualification Tool User Guide](https://docs.nvidia.com/spark-rapids/user-guide/latest/qualification/quickstart.html#running-the-tool).\n", "\n", "### Note\n", - "- Currently, only local or DBFS event log paths are supported.\n", - "- The DBFS path must use the File API format. Example: `/dbfs/`.\n", - "- Multiple event logs must be comma-separated.\n" + "- Currently, local, S3 or DBFS event log paths are supported.\n", + "- S3 path is only supported on Databricks AWS using [instance profiles](https://docs.databricks.com/en/connect/storage/tutorial-s3-instance-profile.html).\n", + "- Eventlog path must follow the formats `/dbfs/path/to/eventlog` or `dbfs:/path/to/eventlog` for logs stored in DBFS.\n", + "- Use wildcards for nested lookup of eventlogs. \n", + " - For example: `/dbfs/path/to/clusterlogs/*/*`\n", + "- Multiple event logs must be comma-separated. \n", + " - For example: `/dbfs/path/to/eventlog1,/dbfs/path/to/eventlog2`\n" ] }, { @@ -45,7 +49,7 @@ }, "outputs": [], "source": [ - "TOOLS_VER = \"24.06.0\"\n", + "TOOLS_VER = \"24.08.0\"\n", "print(f\"Using Tools Version: {TOOLS_VER}\")" ] }, @@ -92,16 +96,28 @@ "import os\n", "import pandas as pd\n", "\n", - "# Initialize variables from widgets\n", - "dbutils.widgets.dropdown(\"Cloud Provider\", \"aws\", [\"aws\", \"azure\"])\n", - "CSP=dbutils.widgets.get(\"Cloud Provider\")\n", "\n", + "def convert_dbfs_path(path):\n", + " return path.replace(\"dbfs:/\", \"/dbfs/\")\n", + " \n", + "# Detect cloud provider from cluster usage tags\n", + "valid_csps = [\"aws\", \"azure\"]\n", + "CSP=spark.conf.get(\"spark.databricks.clusterUsageTags.cloudProvider\", \"\").lower()\n", + "if CSP not in valid_csps:\n", + " print(f\"ERROR: Cannot detect cloud provider from cluster usage tags. Using '{valid_csps[0]}' as default. \")\n", + " CSP = valid_csps[0]\n", + "else:\n", + " print(f\"Detected Cloud Provider from Spark Configs: '{CSP}'\")\n", + "\n", + "# Initialize variables from widgets\n", "dbutils.widgets.text(\"Eventlog Path\", \"/dbfs/user1/qualification_logs\")\n", "EVENTLOG_PATH=dbutils.widgets.get(\"Eventlog Path\")\n", + "EVENTLOG_PATH=convert_dbfs_path(EVENTLOG_PATH)\n", "\n", "dbutils.widgets.text(\"Output Path\", \"/tmp\")\n", "OUTPUT_PATH=dbutils.widgets.get(\"Output Path\")\n", "\n", + " \n", "# Setup environment variables\n", "os.environ[\"CSP\"] = CSP\n", "os.environ[\"EVENTLOG_PATH\"] = EVENTLOG_PATH\n", @@ -133,7 +149,7 @@ "outputs": [], "source": [ "%sh\n", - "spark_rapids qualification --platform databricks-$CSP --eventlogs $EVENTLOG_PATH -o $OUTPUT_PATH > $CONSOLE_OUTPUT_PATH 2> $CONSOLE_ERROR_PATH" + "spark_rapids qualification --platform databricks-$CSP --eventlogs \"$EVENTLOG_PATH\" -o \"$OUTPUT_PATH\" --verbose > \"$CONSOLE_OUTPUT_PATH\" 2> \"$CONSOLE_ERROR_PATH\"" ] }, { @@ -152,9 +168,7 @@ }, "source": [ "## Console Output\n", - "Console output shows the top candidates and their estimated GPU speedup.\n", - "\n", - "**Note**: Use the `--verbose` flag in the command above for more detailed output.\n" + "Console output shows the top candidates and their estimated GPU speedup.\n" ] }, { @@ -193,7 +207,7 @@ "inputWidgets": {}, "nuid": "f3c68b28-fc62-40ae-8528-799f3fc7507e", "showTitle": true, - "title": "Show Error Log" + "title": "Show Logs" }, "jupyter": { "source_hidden": true @@ -268,6 +282,7 @@ "\n", "try:\n", " output_folder, log_file_location = extract_file_info(CONSOLE_OUTPUT_PATH, OUTPUT_PATH)\n", + " jar_output_folder = os.path.join(output_folder, \"rapids_4_spark_qualification_output\")\n", " print(f\"Output folder detected {output_folder}\")\n", " copy_logs(output_folder, log_file_location, CONSOLE_OUTPUT_PATH, CONSOLE_ERROR_PATH)\n", " print(f\"Logs successfully copied to {output_folder}\")\n", @@ -410,9 +425,110 @@ "outputs": [], "source": [ "summary_output=pd.read_csv(os.path.join(output_folder, \"qualification_summary.csv\"))\n", + "summary_output=summary_output.drop(columns=[\"Unnamed: 0\"]).rename_axis('Index').reset_index()\n", "display(summary_output)" ] }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "73b5e0b0-3a96-4cc6-8e6c-840e4b0d9d43", + "showTitle": false, + "title": "" + } + }, + "source": [ + "\n", + "## Application Status\n", + "\n", + "The report show the status of each eventlog file that was provided\n" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "c9ffbfdb-dbb6-4736-b9cb-2ac457cc6714", + "showTitle": true, + "title": "rapids_4_spark_qualification_output_status.csv" + }, + "jupyter": { + "source_hidden": true + } + }, + "outputs": [], + "source": [ + "status_output=pd.read_csv(os.path.join(jar_output_folder, \"rapids_4_spark_qualification_output_status.csv\"))\n", + "display(status_output)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "09945d39-f9c2-4f4a-8afd-4f309f24f8e0", + "showTitle": false, + "title": "" + } + }, + "source": [ + "\n", + "## Metadata for Migration\n", + "\n", + "The report show the metadata of each app as:\n", + "- Recommended GPU cluster\n", + "- File location of full cluster config recommendations\n", + "- File location of only Gpu specific config recommendations\n" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "133cf1bd-33b6-4a62-9ae2-5505717092d1", + "showTitle": true, + "title": "app_metadata.json" + }, + "jupyter": { + "source_hidden": true + } + }, + "outputs": [], + "source": [ + "import json\n", + "metadata_file = os.path.join(output_folder, \"app_metadata.json\")\n", + "def camel_to_title(name):\n", + " return re.sub('([a-z])([A-Z])', r'\\1 \\2', name).title()\n", + " \n", + "with open(metadata_file, 'r') as file:\n", + " json_data = json.load(file)\n", + "\n", + "df = pd.DataFrame(json_data)\n", + "df['recommendedGpuCluster'] = df['clusterInfo'].apply(lambda x: x['recommendedCluster'])\n", + "df['sourceCluster'] = df['clusterInfo'].apply(lambda x: x['sourceCluster'])\n", + "df.drop(columns=['clusterInfo'], inplace=True)\n", + "df = df[['appId', 'appName', 'estimatedGpuSpeedupCategory', 'recommendedGpuCluster', 'fullClusterConfigRecommendations', 'gpuConfigRecommendationBreakdown']]\n", + "df.columns = [camel_to_title(col) for col in df.columns]\n", + "display(df)" + ] + }, { "cell_type": "markdown", "metadata": { @@ -460,7 +576,6 @@ }, "outputs": [], "source": [ - "jar_output_folder = os.path.join(output_folder, \"rapids_4_spark_qualification_output\")\n", "stages_output=pd.read_csv(os.path.join(jar_output_folder, \"rapids_4_spark_qualification_output_stages.csv\"))\n", "display(stages_output)" ] @@ -510,7 +625,7 @@ "inputWidgets": {}, "nuid": "998b0c51-0cb6-408e-a01a-d1f5b1a61e1f", "showTitle": true, - "title": "rapids_4_spark_qualification_output_execs" + "title": "rapids_4_spark_qualification_output_execs.csv" }, "jupyter": { "source_hidden": true @@ -535,7 +650,7 @@ "stack": true }, "nuid": "91c1bfb2-695a-4e5c-8a25-848a433108dc", - "origId": 825198511668429, + "origId": 2173122769183715, "title": "Executive View", "version": "DashboardViewV1", "width": 1600 @@ -549,61 +664,51 @@ "stack": true }, "nuid": "62243296-4562-4f06-90ac-d7a609f19c16", - "origId": 825198511668430, + "origId": 2173122769183716, "title": "App View", "version": "DashboardViewV1", "width": 1920 + }, + { + "elements": [], + "globalVars": {}, + "guid": "", + "layoutOption": { + "grid": true, + "stack": true + }, + "nuid": "854f9c75-5977-42aa-b3dd-c680b8331f19", + "origId": 2173122769183722, + "title": "Untitled", + "version": "DashboardViewV1", + "width": 1024 } ], "environmentMetadata": null, "language": "python", "notebookMetadata": { "mostRecentlyExecutedCommandWithImplicitDF": { - "commandId": 825198511668420, + "commandId": 2173122769183704, "dataframes": [ "_sqldf" ] }, "pythonIndentUnit": 2, "widgetLayout": [ - { - "breakBefore": false, - "name": "Cloud Provider", - "width": 183 - }, { "breakBefore": false, "name": "Eventlog Path", - "width": 728 + "width": 778 }, { "breakBefore": false, "name": "Output Path", - "width": 232 + "width": 302 } ] }, "notebookName": "[RAPIDS Accelerator for Apache Spark] Qualification Tool Notebook Template", "widgets": { - "Cloud Provider": { - "currentValue": "aws", - "nuid": "8dddcaf7-104e-4247-b811-ff7a133b28d4", - "typedWidgetInfo": null, - "widgetInfo": { - "widgetType": "dropdown", - "defaultValue": "aws", - "label": null, - "name": "Cloud Provider", - "options": { - "widgetType": "dropdown", - "autoCreated": null, - "choices": [ - "aws", - "azure" - ] - } - } - }, "Eventlog Path": { "currentValue": "/dbfs/user1/qualification_logs", "nuid": "1272501d-5ad9-42be-ab62-35768b2fc384", @@ -611,11 +716,11 @@ "widgetInfo": { "widgetType": "text", "defaultValue": "/dbfs/user1/qualification_logs", - "label": null, + "label": "", "name": "Eventlog Path", "options": { "widgetType": "text", - "autoCreated": null, + "autoCreated": false, "validationRegex": null } }