Install python dependencies required for running tests in the built venv

test/ci/buildkite/ngtf-code-format.yml

@@ -1,6 +1,7 @@
   - command: |
       source /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/venv/bin/activate 
       export PATH=/opt/llvm-3.9.0/bin/:$$PATH
+      pip install yapf==0.26.0
       maint/check-code-format.sh
 
     label: ":pencil: Code Format ?"

test/ci/buildkite/ngtf-cpu_model-test.yaml

@@ -24,9 +24,8 @@
 
   - command: |
       source /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/venv/bin/activate 
-      pip install psutil && pip install -U \
+      pip install -U \
         /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/tensorflow/tensorflow-2.2.0-cp36-cp36m-linux_x86_64.whl
-      pip install -U pip==19.3.1
       pip install -U /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/ngraph_tensorflow_bridge-*.whl
 
     label: ":gear: Install"

test/ci/buildkite/setup.yml

@@ -7,8 +7,7 @@ steps:
       rm -rf /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID
       virtualenv -p python3 /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/venv 
       source /localdisk/buildkite/artifacts/$BUILDKITE_BUILD_ID/venv/bin/activate 
-      pip install -U pip yapf==0.26.0 pytest psutil keras==2.3.1
-
+
     label: ":gear: Setup"
     timeout_in_minutes: 30
     parallelism: 1

tools/build_utils.py

@@ -25,6 +25,7 @@
 import sys
 import shutil
 import glob
+import multiprocessing
 import platform
 import shlex
 
@@ -104,13 +105,12 @@ def cmake_build(build_dir, src_location, cmake_flags, verbose):
 
     command_executor(cmake_cmd, verbose=True)
 
-    import psutil
-    num_cores = str(psutil.cpu_count(logical=True))
+    num_cores = str(multiprocessing.cpu_count())
     cmd = ["make", "-j" + num_cores]
     if verbose:
         cmd.extend(['VERBOSE=1'])
     command_executor(cmd, verbose=True)
-    cmd = ["make", "install"]
+    cmd = ["make", "-j", "install"]
     command_executor(cmd, verbose=True)
 
     os.chdir(pwd)
@@ -168,7 +168,6 @@ def setup_venv(venv_dir):
         "install",
         "-U",
         "pip",
-        "psutil",
         "six>=1.12.0",
         "numpy>=1.16.0,<1.19.0",
         "wheel>=0.26",
@@ -180,6 +179,8 @@ def setup_venv(venv_dir):
         "keras_preprocessing>=1.1.1,<1.2",
         "--no-deps",
         "yapf==0.26.0",
+        "pytest",
+        "keras==2.3.1",
     ]
     command_executor(package_list)
 
@@ -438,9 +439,7 @@ def build_ngraph_tf(build_dir, artifacts_location, ngtf_src_loc, venv_dir,
     cmake_cmd.extend([ngtf_src_loc])
     command_executor(cmake_cmd)
 
-    import psutil
-    num_cores = str(psutil.cpu_count(logical=True))
-    make_cmd = ["make", "-j" + num_cores, "install"]
+    make_cmd = ["make", "-j", "install"]
     if verbose:
         make_cmd.extend(['VERBOSE=1'])
 

tools/test_utils.py

@@ -24,6 +24,7 @@
 import glob
 import platform
 import subprocess
+import multiprocessing
 from distutils.sysconfig import get_python_lib
 
 from tools.build_utils import load_venv, command_executor, apply_patch
@@ -147,17 +148,10 @@ def run_ngtf_pytests_from_artifacts(artifacts_dir):
 
     if not os.path.isdir(test_dir):
         raise Exception("test directory doesn't exist: " + test_dir)
-
-    # Change the directory to the test_dir
     os.chdir(test_dir)
 
-    # Next run the ngraph-tensorflow python tests
-    command_executor(["pip", "install", "-U", "pytest"])
-    command_executor(["pip", "install", "-U", "psutil"])
-
-    test_manifest_file = TestEnv.get_test_manifest_filename()
     # export the env-var for pytest to process manifest in conftest.py
-    os.environ['NGRAPH_TF_TEST_MANIFEST'] = test_manifest_file
+    os.environ['NGRAPH_TF_TEST_MANIFEST'] = TestEnv.get_test_manifest_filename()
 
     command_executor([
         "python", "-m", "pytest",
@@ -211,12 +205,10 @@ def run_tensorflow_pytests_from_artifacts(ngraph_tf_src_dir, tf_src_dir,
 
     test_xml_report = './junit_tensorflow_tests.xml'
 
-    import psutil
-    num_cores = int(psutil.cpu_count(logical=False))
-    print("OMP_NUM_THREADS: %s " % str(num_cores))
-    os.environ['OMP_NUM_THREADS'] = str(num_cores)
+    num_cores = str(multiprocessing.cpu_count())
+    print("OMP_NUM_THREADS: %s " % num_cores)
+    os.environ['OMP_NUM_THREADS'] = num_cores
     os.environ['NGRAPH_TF_DISABLE_DEASSIGN_CLUSTERS'] = '1'
-
     cmd = [
         "python", test_script, "--tensorflow_path", tf_src_dir,
         "--run_tests_from_file", test_manifest_file
@@ -258,11 +250,9 @@ def run_resnet50_from_artifacts(ngraph_tf_src_dir, artifact_dir, batch_size,
         call(['echo', 'import ngraph_bridge'], stdout=outfile)
 
     # Setup the env flags
-    import psutil
-    num_cores = int(psutil.cpu_count(logical=False))
-    print("OMP_NUM_THREADS: %s " % str(num_cores))
-
-    os.environ['OMP_NUM_THREADS'] = str(num_cores)
+    num_cores = str(multiprocessing.cpu_count())
+    print("OMP_NUM_THREADS: %s " % num_cores)
+    os.environ['OMP_NUM_THREADS'] = num_cores
     os.environ["KMP_AFFINITY"] = 'granularity=fine,compact,1,0'
 
     # Delete the temporary model save directory
@@ -321,10 +311,9 @@ def run_resnet50_infer_from_artifacts(artifact_dir, batch_size, iterations):
         print("Using existing pretrained model file: " + pretrained_model)
 
     # Setup the env flags
-    import psutil
-    num_cores = int(psutil.cpu_count(logical=False))
-    print("OMP_NUM_THREADS: %s " % str(num_cores))
-    os.environ['OMP_NUM_THREADS'] = str(num_cores)
+    num_cores = str(multiprocessing.cpu_count())
+    print("OMP_NUM_THREADS: %s " % num_cores)
+    os.environ['OMP_NUM_THREADS'] = num_cores
     os.environ["KMP_AFFINITY"] = 'granularity=fine,compact,1,0'
 
     os.chdir(root_pwd)