diff --git a/main/tools/container/singularity/commands b/main/tools/container/singularity/commands index 46abe6b9..cb522dbe 100644 --- a/main/tools/container/singularity/commands +++ b/main/tools/container/singularity/commands @@ -1,4 +1,5 @@ +# aipanda180 # cp -r /eos/user/w/wguan/idds_ml/singularity/* /opt/singularity cd /opt/singularity/ diff --git a/main/tools/container/singularity/idds_ml_ax_al9.def b/main/tools/container/singularity/idds_ml_ax_al9.def new file mode 100644 index 00000000..a420dba5 --- /dev/null +++ b/main/tools/container/singularity/idds_ml_ax_al9.def @@ -0,0 +1,32 @@ +Bootstrap: docker +From: almalinux:9.2 + +%post + # yum update -q -y + yum install -q -y wget make git gcc openssl-devel bzip2-devel libffi-devel which pip + + ln -s /usr/bin/python3 /usr/bin/python + + pip install --upgrade pip + pip install nevergrad + pip install theano keras h5py matplotlib tabulate + pip install bayesian-optimization + pip install xgboost + pip install lightgbm + pip install ax-platform + + pip install torch pandas numpy matplotlib wandb botorch + + # clean the cache + rm -fr /root/.cache + +%environment + # export LC_ALL=C + # export PATH=/usr/games:$PATH + +%labels + Maintainer iDDS_HPO_Nevergrad(wen.guan@cern.ch) + Version v1.0 + +%runscript + echo "iDDS ML hyper parameter optimization plugin" diff --git a/main/tools/container/singularity/ml_command b/main/tools/container/singularity/ml_command new file mode 100644 index 00000000..b27baa56 --- /dev/null +++ b/main/tools/container/singularity/ml_command @@ -0,0 +1,62 @@ +[root@aipanda180 singularity]# apptainer remote add --no-login SylabsCloud cloud.sylabs.io +INFO: Remote "SylabsCloud" added. + +[root@aipanda180 singularity]# apptainer remote login +Generate an access token at https://cloud.sylabs.io/auth/tokens, and paste it here. +Token entered will be hidden for security. +Access Token: +INFO: Access Token Verified! +INFO: Token stored in /root/.apptainer/remote.yaml + + +[root@aipanda180 singularity]# singularity keys newpair +Enter your name (e.g., John Doe) : Wen Guan +Enter your email address (e.g., john.doe@example.com) : wguan.icedew@gmail.com +Enter optional comment (e.g., development keys) : dev +Enter a passphrase : +Retype your passphrase : +WARNING: passphrases do not match +Enter a passphrase : +Retype your passphrase : +Generating Entity and OpenPGP Key Pair... done +[root@aipanda180 singularity]# singularity keys list +Public key listing (/root/.apptainer/keys/pgp-public): + +0) User: Wen Guan (dev) + Creation time: 2024-04-02 11:47:04 +0200 CEST + Fingerprint: 11B3BCA23474A37E2BB72F8EAD61E4FD656ABA65 + Length (in bits): 4096 + + +[root@aipanda180 singularity]# singularity keys push 11B3BCA23474A37E2BB72F8EAD61E4FD656ABA65 +public key `11B3BCA23474A37E2BB72F8EAD61E4FD656ABA65' pushed to server successfully +[root@aipanda180 singularity]# ls /root/.apptainer/ +cache keys remote-cache remote.yaml +[root@aipanda180 singularity]# ls /root/.apptainer/keys/ +pgp-public pgp-secret + + +[root@aipanda180 singularity]# singularity sign idds_ml_ax_al9.simg +INFO: Signing image with PGP key material +Enter key passphrase : +INFO: Signature created and applied to image 'idds_ml_ax_al9.simg' +[root@aipanda180 singularity]# singularity verify idds_ml_ax_al9.simg +INFO: Verifying image with PGP key material +[LOCAL] Signing entity: Wen Guan (dev) +[LOCAL] Fingerprint: 11B3BCA23474A37E2BB72F8EAD61E4FD656ABA65 +Objects verified: +ID |GROUP |LINK |TYPE +------------------------------------------------ +1 |1 |NONE |Def.FILE +2 |1 |NONE |JSON.Generic +3 |1 |NONE |FS +INFO: Verified signature(s) from image 'idds_ml_ax_al9.simg' + + +[root@aipanda180 singularity]# cp /root/.apptainer/ +cache/ keys/ remote-cache/ remote.yaml +[root@aipanda180 singularity]# cp /root/.apptainer/remote.yaml /afs/cern.ch/user/w/wguan/private/apptainer/ +[root@aipanda180 singularity]# cp -r /root/.apptainer/keys /afs/cern.ch/user/w/wguan/private/apptainer/ + + +[root@aipanda180 singularity]# singularity push idds_ml_ax_al9.simg library://wguanicedew/ml/idds_ml_ax_al9.sif:latest diff --git a/main/tools/env/setup_panda.sh b/main/tools/env/setup_panda.sh index 6c18c1e0..a379dbea 100644 --- a/main/tools/env/setup_panda.sh +++ b/main/tools/env/setup_panda.sh @@ -71,12 +71,12 @@ else export PANDA_CONFIG_ROOT=~/.panda/ # export IDDS_HOST=https://aipanda015.cern.ch:443/idds - + # dev - export IDDS_HOST=https://aipanda104.cern.ch:443/idds + # export IDDS_HOST=https://aipanda104.cern.ch:443/idds # doma - # export IDDS_HOST=https://aipanda105.cern.ch:443/idds + export IDDS_HOST=https://aipanda105.cern.ch:443/idds # export IDDS_BROKERS=atlas-test-mb.cern.ch:61013 # export IDDS_BROKER_DESTINATION=/topic/doma.idds diff --git a/monitor/data/conf.js b/monitor/data/conf.js index b10e8434..dee49b18 100644 --- a/monitor/data/conf.js +++ b/monitor/data/conf.js @@ -1,9 +1,9 @@ var appConfig = { - 'iddsAPI_request': "https://lxplus964.cern.ch:443/idds/monitor_request/null/null", - 'iddsAPI_transform': "https://lxplus964.cern.ch:443/idds/monitor_transform/null/null", - 'iddsAPI_processing': "https://lxplus964.cern.ch:443/idds/monitor_processing/null/null", - 'iddsAPI_request_detail': "https://lxplus964.cern.ch:443/idds/monitor/null/null/true/false/false", - 'iddsAPI_transform_detail': "https://lxplus964.cern.ch:443/idds/monitor/null/null/false/true/false", - 'iddsAPI_processing_detail': "https://lxplus964.cern.ch:443/idds/monitor/null/null/false/false/true" + 'iddsAPI_request': "https://lxplus998.cern.ch:443/idds/monitor_request/null/null", + 'iddsAPI_transform': "https://lxplus998.cern.ch:443/idds/monitor_transform/null/null", + 'iddsAPI_processing': "https://lxplus998.cern.ch:443/idds/monitor_processing/null/null", + 'iddsAPI_request_detail': "https://lxplus998.cern.ch:443/idds/monitor/null/null/true/false/false", + 'iddsAPI_transform_detail': "https://lxplus998.cern.ch:443/idds/monitor/null/null/false/true/false", + 'iddsAPI_processing_detail': "https://lxplus998.cern.ch:443/idds/monitor/null/null/false/false/true" } diff --git a/workflow/tools/env/environment.yml b/workflow/tools/env/environment.yml index 78877365..b59b2e54 100644 --- a/workflow/tools/env/environment.yml +++ b/workflow/tools/env/environment.yml @@ -5,4 +5,5 @@ dependencies: - pip: - anytree - networkx - - idds-common==2.0.9 \ No newline at end of file + - stomp.py==8.0.1 + - idds-common==2.0.9 diff --git a/workflow/tools/make/environment.yaml b/workflow/tools/make/environment.yaml index 7c5ce80f..f1c8273a 100644 --- a/workflow/tools/make/environment.yaml +++ b/workflow/tools/make/environment.yaml @@ -11,7 +11,7 @@ dependencies: - packaging - anytree - networkx - - stomp.py + - stomp.py==8.0.1 - panda-client - cffi - charset_normalizer