diff --git a/test/ci/buildkite/ngtf-code-format.yml b/test/ci/buildkite/ngtf-code-format.yml index 98d9feac7..9e93b47bc 100644 --- a/test/ci/buildkite/ngtf-code-format.yml +++ b/test/ci/buildkite/ngtf-code-format.yml @@ -1,6 +1,7 @@ - command: | source /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/venv/bin/activate export PATH=/opt/llvm-3.9.0/bin/:$$PATH + pip install yapf==0.26.0 maint/check-code-format.sh label: ":pencil: Code Format ?" diff --git a/test/ci/buildkite/ngtf-cpu_macos.yaml b/test/ci/buildkite/ngtf-cpu_macos.yaml deleted file mode 100644 index 426b6aa46..000000000 --- a/test/ci/buildkite/ngtf-cpu_macos.yaml +++ /dev/null @@ -1,96 +0,0 @@ - - command: | - rm -rf /usr/local/var/buildkite-agent/artifacts/$BUILDKITE_BUILD_ID - virtualenv -p /usr/local/miniconda3/bin/python3 /usr/local/var/buildkite-agent/artifacts/$BUILDKITE_BUILD_ID/venv - source /usr/local/var/buildkite-agent/artifacts/$BUILDKITE_BUILD_ID/venv/bin/activate - - label: ":gear: Setup" - timeout_in_minutes: 30 - agents: - - "queue=mac" - parallelism: 1 - - - wait - - - command: | - source /usr/local/var/buildkite-agent/artifacts/$BUILDKITE_BUILD_ID/venv/bin/activate - python3 build_ngtf.py --artifacts /usr/local/var/buildkite-agent/artifacts/$BUILDKITE_BUILD_ID \ - --use_tensorflow_from_location /usr/local/var/buildkite-agent/tf_1.15.2_install - - label: ":hammer_and_wrench: Build" - timeout_in_minutes: 60 - agents: - - "queue=mac" - parallelism: 1 - - - wait - - - command: | - PYTHONPATH=`pwd` python3 test/ci/buildkite/test_runner.py \ - --artifacts /usr/local/var/buildkite-agent/artifacts/$BUILDKITE_BUILD_ID --test_cpp - - label: ":chrome: C++ Unit Test" - timeout_in_minutes: 30 - agents: - - "queue=mac" - - - wait - - - command: | - source /usr/local/var/buildkite-agent/artifacts/$BUILDKITE_BUILD_ID/venv/bin/activate - pip install -U \ - /usr/local/var/buildkite-agent/tf_1.15.2_install/artifacts/tensorflow/tensorflow-1.15.2-cp37-cp37m-macosx_10_14_x86_64.whl - pip install psutil && \ - pip install psutil pip==19.3.1 && \ - pip install -U /usr/local/var/buildkite-agent/artifacts/$BUILDKITE_BUILD_ID/ngraph_tensorflow_bridge-*.whl - - label: ":gear: Install" - timeout_in_minutes: 30 - agents: - - "queue=mac" - - - wait - - command: | - source /usr/local/var/buildkite-agent/artifacts/$BUILDKITE_BUILD_ID/venv/bin/activate - pip install pytest - pip install keras - PYTHONPATH=`pwd`:`pwd`/examples:`pwd`/tools:`pwd`/examples/mnist python3 test/ci/buildkite/test_runner.py \ - --artifacts /usr/local/var/buildkite-agent/artifacts/$BUILDKITE_BUILD_ID --test_python - - label: ":python: nGraph Pytest" - timeout_in_minutes: 60 - soft_fail: true - agents: - - "queue=mac" - - - wait - - command: | - source /usr/local/var/buildkite-agent/artifacts/$BUILDKITE_BUILD_ID/venv/bin/activate - pip install pytest - PYTHONPATH=`pwd` python3 test/ci/buildkite/test_runner.py \ - --artifacts /usr/local/var/buildkite-agent/artifacts/$BUILDKITE_BUILD_ID --test_tf_python - - label: ":python: TensorFlow Pytest" - timeout_in_minutes: 90 - soft_fail: true - agents: - - "queue=mac" - - - wait - - - command: | - source /usr/local/var/buildkite-agent/artifacts/$BUILDKITE_BUILD_ID/venv/bin/activate - PYTHONPATH=`pwd` python3 test/ci/buildkite/test_runner.py \ - --artifacts /usr/local/var/buildkite-agent/artifacts/$BUILDKITE_BUILD_ID --test_resnet - label: ":bar_chart: ResNet50" - timeout_in_minutes: 30 - soft_fail: true - agents: - - "queue=mac" - - - wait: ~ - - command: | - rm -rf /usr/local/var/buildkite-agent/artifacts/$BUILDKITE_BUILD_ID - label: ":wastebasket: Cleanup" - agents: - - "queue=mac" - diff --git a/test/ci/buildkite/ngtf-cpu_model-test.yaml b/test/ci/buildkite/ngtf-cpu_model-test.yaml index e2368ff57..c136f8cfa 100644 --- a/test/ci/buildkite/ngtf-cpu_model-test.yaml +++ b/test/ci/buildkite/ngtf-cpu_model-test.yaml @@ -24,9 +24,8 @@ - command: | source /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/venv/bin/activate - pip install psutil && pip install -U \ + pip install -U \ /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/tensorflow/tensorflow-2.2.0-cp36-cp36m-linux_x86_64.whl - pip install -U pip==19.3.1 pip install -U /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/ngraph_tensorflow_bridge-*.whl label: ":gear: Install" diff --git a/test/ci/buildkite/setup.yml b/test/ci/buildkite/setup.yml index 314e94da0..b1d1449f4 100644 --- a/test/ci/buildkite/setup.yml +++ b/test/ci/buildkite/setup.yml @@ -7,8 +7,7 @@ steps: rm -rf /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID virtualenv -p python3 /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/venv source /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/venv/bin/activate - pip install -U pip yapf==0.26.0 pytest psutil keras==2.3.1 - + label: ":gear: Setup" timeout_in_minutes: 30 parallelism: 1 diff --git a/tools/build_utils.py b/tools/build_utils.py index 6b4604498..29c615821 100755 --- a/tools/build_utils.py +++ b/tools/build_utils.py @@ -25,6 +25,7 @@ import sys import shutil import glob +import multiprocessing import platform import shlex @@ -104,13 +105,12 @@ def cmake_build(build_dir, src_location, cmake_flags, verbose): command_executor(cmake_cmd, verbose=True) - import psutil - num_cores = str(psutil.cpu_count(logical=True)) + num_cores = str(multiprocessing.cpu_count()) cmd = ["make", "-j" + num_cores] if verbose: cmd.extend(['VERBOSE=1']) command_executor(cmd, verbose=True) - cmd = ["make", "install"] + cmd = ["make", "-j", "install"] command_executor(cmd, verbose=True) os.chdir(pwd) @@ -168,7 +168,6 @@ def setup_venv(venv_dir): "install", "-U", "pip", - "psutil", "six>=1.12.0", "numpy>=1.16.0,<1.19.0", "wheel>=0.26", @@ -180,6 +179,8 @@ def setup_venv(venv_dir): "keras_preprocessing>=1.1.1,<1.2", "--no-deps", "yapf==0.26.0", + "pytest", + "keras==2.3.1", ] command_executor(package_list) @@ -438,9 +439,7 @@ def build_ngraph_tf(build_dir, artifacts_location, ngtf_src_loc, venv_dir, cmake_cmd.extend([ngtf_src_loc]) command_executor(cmake_cmd) - import psutil - num_cores = str(psutil.cpu_count(logical=True)) - make_cmd = ["make", "-j" + num_cores, "install"] + make_cmd = ["make", "-j", "install"] if verbose: make_cmd.extend(['VERBOSE=1']) diff --git a/tools/test_utils.py b/tools/test_utils.py index 1bcce7dc2..86302640a 100755 --- a/tools/test_utils.py +++ b/tools/test_utils.py @@ -24,6 +24,7 @@ import glob import platform import subprocess +import multiprocessing from distutils.sysconfig import get_python_lib from tools.build_utils import load_venv, command_executor, apply_patch @@ -147,17 +148,10 @@ def run_ngtf_pytests_from_artifacts(artifacts_dir): if not os.path.isdir(test_dir): raise Exception("test directory doesn't exist: " + test_dir) - - # Change the directory to the test_dir os.chdir(test_dir) - # Next run the ngraph-tensorflow python tests - command_executor(["pip", "install", "-U", "pytest"]) - command_executor(["pip", "install", "-U", "psutil"]) - - test_manifest_file = TestEnv.get_test_manifest_filename() # export the env-var for pytest to process manifest in conftest.py - os.environ['NGRAPH_TF_TEST_MANIFEST'] = test_manifest_file + os.environ['NGRAPH_TF_TEST_MANIFEST'] = TestEnv.get_test_manifest_filename() command_executor([ "python", "-m", "pytest", @@ -211,12 +205,10 @@ def run_tensorflow_pytests_from_artifacts(ngraph_tf_src_dir, tf_src_dir, test_xml_report = './junit_tensorflow_tests.xml' - import psutil - num_cores = int(psutil.cpu_count(logical=False)) - print("OMP_NUM_THREADS: %s " % str(num_cores)) - os.environ['OMP_NUM_THREADS'] = str(num_cores) + num_cores = str(multiprocessing.cpu_count()) + print("OMP_NUM_THREADS: %s " % num_cores) + os.environ['OMP_NUM_THREADS'] = num_cores os.environ['NGRAPH_TF_DISABLE_DEASSIGN_CLUSTERS'] = '1' - cmd = [ "python", test_script, "--tensorflow_path", tf_src_dir, "--run_tests_from_file", test_manifest_file @@ -258,11 +250,9 @@ def run_resnet50_from_artifacts(ngraph_tf_src_dir, artifact_dir, batch_size, call(['echo', 'import ngraph_bridge'], stdout=outfile) # Setup the env flags - import psutil - num_cores = int(psutil.cpu_count(logical=False)) - print("OMP_NUM_THREADS: %s " % str(num_cores)) - - os.environ['OMP_NUM_THREADS'] = str(num_cores) + num_cores = str(multiprocessing.cpu_count()) + print("OMP_NUM_THREADS: %s " % num_cores) + os.environ['OMP_NUM_THREADS'] = num_cores os.environ["KMP_AFFINITY"] = 'granularity=fine,compact,1,0' # Delete the temporary model save directory @@ -321,10 +311,9 @@ def run_resnet50_infer_from_artifacts(artifact_dir, batch_size, iterations): print("Using existing pretrained model file: " + pretrained_model) # Setup the env flags - import psutil - num_cores = int(psutil.cpu_count(logical=False)) - print("OMP_NUM_THREADS: %s " % str(num_cores)) - os.environ['OMP_NUM_THREADS'] = str(num_cores) + num_cores = str(multiprocessing.cpu_count()) + print("OMP_NUM_THREADS: %s " % num_cores) + os.environ['OMP_NUM_THREADS'] = num_cores os.environ["KMP_AFFINITY"] = 'granularity=fine,compact,1,0' os.chdir(root_pwd)