BLD: Fix slurm CI/CD workflow (xorbitsai#801)

Co-authored-by: Dawnfz-lenfeng <[email protected]>

CI/slurm/Dockerfile

@@ -1,2 +1,7 @@
 FROM daskdev/dask-jobqueue:slurm
-RUN pip install xorbits
+
+SHELL ["/bin/bash", "-c"]
+
+# activate conda env
+RUN echo "source /opt/anaconda/bin/activate dask-jobqueue" >> ~/.bashrc
+ENV PATH /opt/anaconda/envs/dask-jobqueue/bin:$PATH

CI/slurm/docker-compose.yml

@@ -16,8 +16,7 @@ services:
  common-network:
 
  slurmdbd:
- image: daskdev/dask-jobqueue:slurm
- build: .
+ image: slurmbase
  command: ["slurmdbd"]
  container_name: slurmdbd
  hostname: slurmdbd
@@ -33,8 +32,7 @@ services:
  common-network:
 
  slurmctld:
- image: daskdev/dask-jobqueue:slurm
- build: .
+ image: slurmbase
  command: ["slurmctld"]
  container_name: slurmctld
  hostname: slurmctld
@@ -58,8 +56,7 @@ services:
  - NET_ADMIN
 
  c1:
- image: daskdev/dask-jobqueue:slurm
- build: .
+ image: slurmbase
  command: ["slurmd"]
  hostname: c1
  container_name: c1
@@ -81,8 +78,7 @@ services:
  - NET_ADMIN
 
  c2:
- image: daskdev/dask-jobqueue:slurm
- build: .
+ image: slurmbase
  command: ["slurmd"]
  hostname: c2
  container_name: c2

CI/slurm/register_cluster.sh

@@ -2,4 +2,4 @@
 set -e
 
 docker exec slurmctld bash -c "/usr/bin/sacctmgr --immediate add cluster name=linux" && \
-docker-compose restart slurmdbd slurmctld
+docker compose restart slurmdbd slurmctld

CI/slurm/slurm.sh

@@ -15,15 +15,15 @@
 
 function jobqueue_before_install {
  docker version
- docker-compose version
+ docker compose version
 
  # start slurm cluster
  cd ./CI/slurm
- docker-compose pull
+ docker build -t slurmbase .
  ./start-slurm.sh
  cd -
 
- #Set shared space permissions
+ # set shared space permissions
  docker exec slurmctld /bin/bash -c "chmod -R 777 /shared_space"
 
  docker ps -a
@@ -35,6 +35,7 @@ function show_network_interfaces {
  for c in slurmctld c1 c2; do
  echo '------------------------------------------------------------'
  echo docker container: $c
+ docker exec $c pip install psutil
  docker exec $c python -c 'import psutil; print(psutil.net_if_addrs().keys())'
  echo '------------------------------------------------------------'
  done

CI/slurm/start-slurm.sh

@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-docker-compose up -d --no-build
+docker compose up -d --no-build
 
 while [ `./register_cluster.sh 2>&1 | grep "sacctmgr: error" | wc -l` -ne 0 ]
  do