[BugFix] Higher time threshold for PEnv (#1799)

pytorch · Jan 14, 2024 · 66d5a00 · 66d5a00
1 parent 95d8a1d
commit 66d5a00
Show file tree

Hide file tree

Showing 8 changed files with 22 additions and 2 deletions.
diff --git a/.github/unittest/linux/scripts/run_all.sh b/.github/unittest/linux/scripts/run_all.sh
@@ -179,6 +179,7 @@ python -m torch.utils.collect_env
 export MKL_THREADING_LAYER=GNU
 export CKPT_BACKEND=torch
 export MAX_IDLE_COUNT=100
+export BATCHED_PIPE_TIMEOUT=60
 
 pytest test/smoke_test.py -v --durations 200
 pytest test/smoke_test_deps.py -v --durations 200 -k 'test_gym or test_dm_control_pixels or test_dm_control or test_tb'

diff --git a/.github/unittest/linux_distributed/scripts/run_test.sh b/.github/unittest/linux_distributed/scripts/run_test.sh
@@ -18,6 +18,7 @@ lib_dir="${env_dir}/lib"
 export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$lib_dir
 export MKL_THREADING_LAYER=GNU
 export CKPT_BACKEND=torch
+export BATCHED_PIPE_TIMEOUT=60
 
 python .github/unittest/helpers/coverage_run_parallel.py -m pytest test/smoke_test.py -v --durations 200
 python .github/unittest/helpers/coverage_run_parallel.py -m pytest test/smoke_test_deps.py -v --durations 200 -k 'test_gym or test_dm_control_pixels or test_dm_control or test_tb'

diff --git a/.github/unittest/linux_examples/scripts/run_all.sh b/.github/unittest/linux_examples/scripts/run_all.sh
@@ -80,7 +80,8 @@ conda env config vars set MUJOCO_PY_MUJOCO_PATH=$root_dir/.mujoco/mujoco210 \
  LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$root_dir/.mujoco/mujoco210/bin \
  SDL_VIDEODRIVER=dummy \
  MUJOCO_GL=egl \
- PYOPENGL_PLATFORM=egl
+ PYOPENGL_PLATFORM=egl \
+ BATCHED_PIPE_TIMEOUT=60
 
 pip install pip --upgrade
 

diff --git a/.github/unittest/linux_olddeps/scripts_gym_0_13/run_test.sh b/.github/unittest/linux_olddeps/scripts_gym_0_13/run_test.sh
@@ -18,6 +18,7 @@ export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$lib_dir
 #export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/work/mujoco-py/mujoco_py/binaries/linux/mujoco210/bin
 export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/pytorch/rl/mujoco-py/mujoco_py/binaries/linux/mujoco210/bin
 export MKL_THREADING_LAYER=GNU
+export BATCHED_PIPE_TIMEOUT=60
 
 python .github/unittest/helpers/coverage_run_parallel.py -m pytest test/smoke_test.py -v --durations 200
 python .github/unittest/helpers/coverage_run_parallel.py -m pytest test/smoke_test_deps.py -v --durations 200 -k 'test_gym or test_dm_control_pixels or test_dm_control'

diff --git a/.github/unittest/linux_optdeps/scripts/run_test.sh b/.github/unittest/linux_optdeps/scripts/run_test.sh
@@ -15,6 +15,7 @@ git config --global --add safe.directory '*'
 root_dir="$(git rev-parse --show-toplevel)"
 export MKL_THREADING_LAYER=GNU
 export CKPT_BACKEND=torch
+export BATCHED_PIPE_TIMEOUT=60
 
 MUJOCO_GL=egl python .github/unittest/helpers/coverage_run_parallel.py -m pytest --instafail -v --durations 200 --ignore test/test_distributed.py --ignore test/test_rlhf.py
 coverage combine

diff --git a/.github/unittest/windows_optdepts/scripts/run_test.sh b/.github/unittest/windows_optdepts/scripts/run_test.sh
@@ -11,6 +11,7 @@ source "$this_dir/set_cuda_envs.sh"
 # we don't use torchsnapshot
 export CKPT_BACKEND=torch
 export MAX_IDLE_COUNT=60
+export BATCHED_PIPE_TIMEOUT=60
 
 python -m torch.utils.collect_env
 pytest --junitxml=test-results/junit.xml -v --durations 200 --ignore test/test_distributed.py --ignore test/test_rlhf.py
diff --git a/.github/workflows/test-linux-libs.yml b/.github/workflows/test-linux-libs.yml
@@ -36,6 +36,7 @@ jobs:
  export TAR_OPTIONS="--no-same-owner"
  export UPLOAD_CHANNEL="nightly"
  export TF_CPP_MIN_LOG_LEVEL=0
+ export BATCHED_PIPE_TIMEOUT=60
 
  nvidia-smi
 
@@ -60,6 +61,7 @@ jobs:
  export TAR_OPTIONS="--no-same-owner"
  export UPLOAD_CHANNEL="nightly"
  export TF_CPP_MIN_LOG_LEVEL=0
+ export BATCHED_PIPE_TIMEOUT=60
 
  bash .github/unittest/linux_libs/scripts_d4rl/setup_env.sh
  bash .github/unittest/linux_libs/scripts_d4rl/install.sh
@@ -85,6 +87,7 @@ jobs:
  export TAR_OPTIONS="--no-same-owner"
  export UPLOAD_CHANNEL="nightly"
  export TF_CPP_MIN_LOG_LEVEL=0
+ export BATCHED_PIPE_TIMEOUT=60
 
  bash .github/unittest/linux_libs/scripts_d4rl/setup_env.sh
  bash .github/unittest/linux_libs/scripts_d4rl/install.sh
@@ -111,6 +114,7 @@ jobs:
  export CU_VERSION="11.4"
  export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/work/mujoco-py/mujoco_py/binaries/linux/mujoco210/bin"
  export TAR_OPTIONS="--no-same-owner"
+ export BATCHED_PIPE_TIMEOUT=60
 
  ./.github/unittest/linux_libs/scripts_gym/setup_env.sh
  ./.github/unittest/linux_libs/scripts_gym/batch_scripts.sh
@@ -135,6 +139,7 @@ jobs:
  export TAR_OPTIONS="--no-same-owner"
  export UPLOAD_CHANNEL="nightly"
  export TF_CPP_MIN_LOG_LEVEL=0
+ export BATCHED_PIPE_TIMEOUT=60
 
  nvidia-smi
 
@@ -162,6 +167,7 @@ jobs:
  export TAR_OPTIONS="--no-same-owner"
  export UPLOAD_CHANNEL="nightly"
  export TF_CPP_MIN_LOG_LEVEL=0
+ export BATCHED_PIPE_TIMEOUT=60
 
  bash .github/unittest/linux_libs/scripts_minari/setup_env.sh
  bash .github/unittest/linux_libs/scripts_minari/install.sh
@@ -187,6 +193,7 @@ jobs:
  export TAR_OPTIONS="--no-same-owner"
  export UPLOAD_CHANNEL="nightly"
  export TF_CPP_MIN_LOG_LEVEL=0
+ export BATCHED_PIPE_TIMEOUT=60
 
  bash .github/unittest/linux_libs/scripts_openx/setup_env.sh
  bash .github/unittest/linux_libs/scripts_openx/install.sh
@@ -208,6 +215,7 @@ jobs:
  export TAR_OPTIONS="--no-same-owner"
  export UPLOAD_CHANNEL="nightly"
  export TF_CPP_MIN_LOG_LEVEL=0
+ export BATCHED_PIPE_TIMEOUT=60
 
  nvidia-smi
 
@@ -234,6 +242,7 @@ jobs:
  export TAR_OPTIONS="--no-same-owner"
  export UPLOAD_CHANNEL="nightly"
  export TF_CPP_MIN_LOG_LEVEL=0
+ export BATCHED_PIPE_TIMEOUT=60
 
  bash .github/unittest/linux_libs/scripts_robohive/setup_env.sh
  bash .github/unittest/linux_libs/scripts_robohive/install_and_run_test.sh
@@ -258,6 +267,7 @@ jobs:
  export TAR_OPTIONS="--no-same-owner"
  export UPLOAD_CHANNEL="nightly"
  export TF_CPP_MIN_LOG_LEVEL=0
+ export BATCHED_PIPE_TIMEOUT=60
 
  bash .github/unittest/linux_libs/scripts_roboset/setup_env.sh
  bash .github/unittest/linux_libs/scripts_roboset/install.sh
@@ -284,6 +294,7 @@ jobs:
  export TAR_OPTIONS="--no-same-owner"
  export UPLOAD_CHANNEL="nightly"
  export TF_CPP_MIN_LOG_LEVEL=0
+ export BATCHED_PIPE_TIMEOUT=60
 
  bash .github/unittest/linux_libs/scripts_sklearn/setup_env.sh
  bash .github/unittest/linux_libs/scripts_sklearn/install.sh
@@ -310,6 +321,7 @@ jobs:
  export TAR_OPTIONS="--no-same-owner"
  export UPLOAD_CHANNEL="nightly"
  export TF_CPP_MIN_LOG_LEVEL=0
+ export BATCHED_PIPE_TIMEOUT=60
 
  nvidia-smi
 
@@ -337,6 +349,7 @@ jobs:
  export TAR_OPTIONS="--no-same-owner"
  export UPLOAD_CHANNEL="nightly"
  export TF_CPP_MIN_LOG_LEVEL=0
+ export BATCHED_PIPE_TIMEOUT=60
 
  bash .github/unittest/linux_libs/scripts_vd4rl/setup_env.sh
  bash .github/unittest/linux_libs/scripts_vd4rl/install.sh
@@ -362,6 +375,7 @@ jobs:
  export TAR_OPTIONS="--no-same-owner"
  export UPLOAD_CHANNEL="nightly"
  export TF_CPP_MIN_LOG_LEVEL=0
+ export BATCHED_PIPE_TIMEOUT=60
 
  nvidia-smi
 

diff --git a/torchrl/_utils.py b/torchrl/_utils.py
@@ -32,7 +32,7 @@
 VERBOSE = strtobool(os.environ.get("VERBOSE", "0"))
 _os_is_windows = sys.platform == "win32"
 RL_WARNINGS = strtobool(os.environ.get("RL_WARNINGS", "1"))
-BATCHED_PIPE_TIMEOUT = float(os.environ.get("RL_WARNINGS", "60.0"))
+BATCHED_PIPE_TIMEOUT = float(os.environ.get("BATCHED_PIPE_TIMEOUT", "10000.0"))
 
 
 class timeit: