From dcd2ae83a6fb52b3dd643d5838f59049ea3890ec Mon Sep 17 00:00:00 2001 From: Haimasree Date: Fri, 10 Jun 2022 10:49:43 +0200 Subject: [PATCH 01/14] adding shared env --- .circleci/config.yml | 4 +++- shared/envs/kipoi-py3-keras1.2.yaml | 4 ++-- shared/envs/kipoi-py3-keras2-tf1.yaml | 4 ++-- shared/envs/kipoi-py3-keras2-tf2.yaml | 4 ++-- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 64a65773c..b71b900df 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -222,13 +222,14 @@ workflows: ignore: - master - test_all + - sharedenvupdate - test_new_models_singularity: filters: branches: ignore: - master - test_all - + - sharedenvupdate test-all-branch: jobs: - test_all_models: @@ -261,6 +262,7 @@ workflows: only: - master - test_all + - sharedenvupdate - sync_containers: context: - sync-kipoi-containers diff --git a/shared/envs/kipoi-py3-keras1.2.yaml b/shared/envs/kipoi-py3-keras1.2.yaml index 76100b3fa..2f15a106f 100644 --- a/shared/envs/kipoi-py3-keras1.2.yaml +++ b/shared/envs/kipoi-py3-keras1.2.yaml @@ -14,14 +14,14 @@ dependencies: - pandas - cython=0.29.28 - pip=21.3.1 +- tensorflow<2 +- keras<2 - pip: - pyvcf==0.4.3 - kipoi_veff - kipoi_interpret - kipoi - kipoiseq - - tensorflow==1.15 - - keras==1.2.2 - deepcpg==1.0.4 - h5py==3.6.0 - sorted-nearest==0.0.33 diff --git a/shared/envs/kipoi-py3-keras2-tf1.yaml b/shared/envs/kipoi-py3-keras2-tf1.yaml index 304d6a817..7fb8f0ad1 100644 --- a/shared/envs/kipoi-py3-keras2-tf1.yaml +++ b/shared/envs/kipoi-py3-keras2-tf1.yaml @@ -22,6 +22,8 @@ dependencies: - pybedtools=0.8.1 - pysam=0.15.3 - joblib=1.1.0 +- tensorflow<2 +- keras==2.1 - pip: - scikit-learn==0.19.2 - gtfparse>=1.0.7 @@ -30,8 +32,6 @@ dependencies: - kipoi_interpret - kipoi - kipoiseq - - tensorflow==1.15 - - keras==2.1.6 - gffutils==0.10.1 - pyfaidx==0.6.4 # - scikit-learn diff --git a/shared/envs/kipoi-py3-keras2-tf2.yaml b/shared/envs/kipoi-py3-keras2-tf2.yaml index 984d16776..ddb109f7b 100644 --- a/shared/envs/kipoi-py3-keras2-tf2.yaml +++ b/shared/envs/kipoi-py3-keras2-tf2.yaml @@ -22,6 +22,8 @@ dependencies: - pysam=0.17 - joblib=1.1.0 - scikit-learn=1.0.2 +- tensorflow +- keras - pip: - gtfparse>=1.0.7 - pyvcf==0.4.3 @@ -29,8 +31,6 @@ dependencies: - kipoi_interpret - kipoi - kipoiseq - - tensorflow - - keras - gffutils==0.10.1 - pyfaidx==0.6.4 # - scikit-learn From dfc84df2e182cad9ca03cb63f8c29b1d67ad8a70 Mon Sep 17 00:00:00 2001 From: Haimasree Date: Fri, 10 Jun 2022 10:55:29 +0200 Subject: [PATCH 02/14] fixing keras version --- shared/envs/kipoi-py3-keras2-tf1.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/envs/kipoi-py3-keras2-tf1.yaml b/shared/envs/kipoi-py3-keras2-tf1.yaml index 7fb8f0ad1..6c17391a4 100644 --- a/shared/envs/kipoi-py3-keras2-tf1.yaml +++ b/shared/envs/kipoi-py3-keras2-tf1.yaml @@ -23,7 +23,7 @@ dependencies: - pysam=0.15.3 - joblib=1.1.0 - tensorflow<2 -- keras==2.1 +- keras=2.1 - pip: - scikit-learn==0.19.2 - gtfparse>=1.0.7 From 0305b1e76412a6506f705e6ed1c2b339cf47df2b Mon Sep 17 00:00:00 2001 From: Haimasree Date: Fri, 10 Jun 2022 11:54:02 +0200 Subject: [PATCH 03/14] bug fix --- shared/envs/kipoi-py3-keras1.2.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/envs/kipoi-py3-keras1.2.yaml b/shared/envs/kipoi-py3-keras1.2.yaml index 2f15a106f..ccff1e83c 100644 --- a/shared/envs/kipoi-py3-keras1.2.yaml +++ b/shared/envs/kipoi-py3-keras1.2.yaml @@ -15,8 +15,8 @@ dependencies: - cython=0.29.28 - pip=21.3.1 - tensorflow<2 -- keras<2 - pip: + - keras==1.2.2 - pyvcf==0.4.3 - kipoi_veff - kipoi_interpret From 67352f17448995e18ed7eb8b520a27b7110d8944 Mon Sep 17 00:00:00 2001 From: Haimasree Date: Fri, 10 Jun 2022 12:00:58 +0200 Subject: [PATCH 04/14] keras1 yaml fixed --- shared/envs/kipoi-py3-keras1.2.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/envs/kipoi-py3-keras1.2.yaml b/shared/envs/kipoi-py3-keras1.2.yaml index ccff1e83c..b52f53955 100644 --- a/shared/envs/kipoi-py3-keras1.2.yaml +++ b/shared/envs/kipoi-py3-keras1.2.yaml @@ -14,7 +14,7 @@ dependencies: - pandas - cython=0.29.28 - pip=21.3.1 -- tensorflow<2 +- tensorflow=1.14 - pip: - keras==1.2.2 - pyvcf==0.4.3 From 47e1d8527d4594fa48d30a0964070843ae8bcc4b Mon Sep 17 00:00:00 2001 From: Haimasree Date: Fri, 10 Jun 2022 12:37:55 +0200 Subject: [PATCH 05/14] adding more changes --- shared/envs/kipoi-py3-keras2-tf1.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shared/envs/kipoi-py3-keras2-tf1.yaml b/shared/envs/kipoi-py3-keras2-tf1.yaml index 6c17391a4..bb2a4b11f 100644 --- a/shared/envs/kipoi-py3-keras2-tf1.yaml +++ b/shared/envs/kipoi-py3-keras2-tf1.yaml @@ -22,9 +22,9 @@ dependencies: - pybedtools=0.8.1 - pysam=0.15.3 - joblib=1.1.0 -- tensorflow<2 -- keras=2.1 +- tensorflow=1.14 - pip: + - keras==2.1.6 - scikit-learn==0.19.2 - gtfparse>=1.0.7 - pyvcf==0.4.3 From 4f58ab8d431c790c9aa0560895146080ded34dc1 Mon Sep 17 00:00:00 2001 From: Haimasree Date: Fri, 10 Jun 2022 13:13:17 +0200 Subject: [PATCH 06/14] adding more changes --- shared/envs/kipoi-py3-keras1.2.yaml | 2 -- shared/envs/kipoi-py3-keras2-tf1.yaml | 5 ----- shared/envs/kipoi-py3-keras2-tf2.yaml | 8 ++------ 3 files changed, 2 insertions(+), 13 deletions(-) diff --git a/shared/envs/kipoi-py3-keras1.2.yaml b/shared/envs/kipoi-py3-keras1.2.yaml index b52f53955..b4c4db16d 100644 --- a/shared/envs/kipoi-py3-keras1.2.yaml +++ b/shared/envs/kipoi-py3-keras1.2.yaml @@ -10,8 +10,6 @@ dependencies: - pysam=0.15.3 - python=3.7 - genomelake==0.1.4 -- numpy -- pandas - cython=0.29.28 - pip=21.3.1 - tensorflow=1.14 diff --git a/shared/envs/kipoi-py3-keras2-tf1.yaml b/shared/envs/kipoi-py3-keras2-tf1.yaml index bb2a4b11f..140f77ace 100644 --- a/shared/envs/kipoi-py3-keras2-tf1.yaml +++ b/shared/envs/kipoi-py3-keras2-tf1.yaml @@ -34,14 +34,9 @@ dependencies: - kipoiseq - gffutils==0.10.1 - pyfaidx==0.6.4 - # - scikit-learn - intervaltree==2.1.0 - # - sklearn-pandas - tqdm==4.64.0 - # - mmsplice - - numpy - concise>=0.6.6 - # other - ipykernel==6.13.0 - sorted-nearest==0.0.33 - protobuf==3.20 \ No newline at end of file diff --git a/shared/envs/kipoi-py3-keras2-tf2.yaml b/shared/envs/kipoi-py3-keras2-tf2.yaml index ddb109f7b..864102ad2 100644 --- a/shared/envs/kipoi-py3-keras2-tf2.yaml +++ b/shared/envs/kipoi-py3-keras2-tf2.yaml @@ -22,8 +22,8 @@ dependencies: - pysam=0.17 - joblib=1.1.0 - scikit-learn=1.0.2 -- tensorflow -- keras +- tensorflow>=2.8 +- keras>=2.2 - pip: - gtfparse>=1.0.7 - pyvcf==0.4.3 @@ -33,13 +33,9 @@ dependencies: - kipoiseq - gffutils==0.10.1 - pyfaidx==0.6.4 - # - scikit-learn - intervaltree==2.1.0 - # - sklearn-pandas - tqdm==4.64.0 - # - mmsplice - concise>=0.6.6 - # other - ipykernel==6.13.0 - sorted-nearest==0.0.33 - protobuf==3.20 \ No newline at end of file From 5ab7cd0c5fce5c349c3d77bd6260f92e680b4197 Mon Sep 17 00:00:00 2001 From: Haimasree Date: Fri, 10 Jun 2022 14:21:05 +0200 Subject: [PATCH 07/14] bug fix (hopefully) --- shared/envs/kipoi-py3-keras2-tf2.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shared/envs/kipoi-py3-keras2-tf2.yaml b/shared/envs/kipoi-py3-keras2-tf2.yaml index d102d1cf4..392cdfbaf 100644 --- a/shared/envs/kipoi-py3-keras2-tf2.yaml +++ b/shared/envs/kipoi-py3-keras2-tf2.yaml @@ -22,8 +22,8 @@ dependencies: - pysam=0.17 - joblib=1.1.0 - scikit-learn=1.0.2 -- tensorflow>=2.8 -- keras>=2.2 +- tensorflow +- keras - pip: - gtfparse>=1.0.7 - pyvcf==0.4.3 From e96c8402ea6910cdce30f7ff2cbdbcf76068f4e5 Mon Sep 17 00:00:00 2001 From: Haimasree Date: Fri, 10 Jun 2022 16:03:33 +0200 Subject: [PATCH 08/14] reverting keras2tf2 --- shared/envs/kipoi-py3-keras2-tf2.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/shared/envs/kipoi-py3-keras2-tf2.yaml b/shared/envs/kipoi-py3-keras2-tf2.yaml index 392cdfbaf..4fe714079 100644 --- a/shared/envs/kipoi-py3-keras2-tf2.yaml +++ b/shared/envs/kipoi-py3-keras2-tf2.yaml @@ -22,8 +22,6 @@ dependencies: - pysam=0.17 - joblib=1.1.0 - scikit-learn=1.0.2 -- tensorflow -- keras - pip: - gtfparse>=1.0.7 - pyvcf==0.4.3 @@ -31,10 +29,16 @@ dependencies: - kipoi_interpret - kipoi - kipoiseq + - tensorflow + - keras - gffutils==0.10.1 - pyfaidx==0.6.4 + # - scikit-learn - intervaltree==2.1.0 + # - sklearn-pandas - tqdm==4.64.0 + # - mmsplice - concise>=0.6.6 + # other - ipykernel==6.13.0 - protobuf==3.20 \ No newline at end of file From e0257c8698407649bc0577056694d25f89222d54 Mon Sep 17 00:00:00 2001 From: Haimasree Date: Mon, 13 Jun 2022 15:58:52 +0200 Subject: [PATCH 09/14] adding correct repository dispatch event formatting --- .circleci/config.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b71b900df..92a25c720 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -73,12 +73,9 @@ variables: sync: &sync run: name: Sync kipoi models and containers repo - command: | - curl -X POST https://api.github.com/repos/kipoi/kipoi-containers/dispatches \ - -H 'Accept: application/vnd.github.v3+json' \ - -u $SYNC_TOKEN \ - --data '{"event_type": "sync"}' - + command: curl -X POST \ + -H "Accept: application/vnd.github.v3+json" \ + -H "Authorization: token $SYNC_TOKEN" https://api.github.com/repos/kipoi/kipoi-containers/dispatches -d '{"event_type":"sync"}' jobs: sync_containers: From b30faa52c6c9ab50d9b3175559a44c8df5d77e7f Mon Sep 17 00:00:00 2001 From: Haimasree Date: Wed, 15 Jun 2022 20:41:09 +0200 Subject: [PATCH 10/14] a possibly working version --- shared/envs/kipoi-py3-keras2-tf2.yaml | 33 ++++++++++++--------------- 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/shared/envs/kipoi-py3-keras2-tf2.yaml b/shared/envs/kipoi-py3-keras2-tf2.yaml index 4fe714079..87e411913 100644 --- a/shared/envs/kipoi-py3-keras2-tf2.yaml +++ b/shared/envs/kipoi-py3-keras2-tf2.yaml @@ -1,6 +1,5 @@ name: kipoi-py3-keras2-tf2 channels: -- pytorch - bioconda - conda-forge - defaults @@ -22,23 +21,21 @@ dependencies: - pysam=0.17 - joblib=1.1.0 - scikit-learn=1.0.2 +- tensorflow +- keras - pip: - - gtfparse>=1.0.7 - - pyvcf==0.4.3 - - kipoi_veff - - kipoi_interpret + # - gtfparse>=1.0.7 + # - pyvcf==0.4.3 - kipoi - kipoiseq - - tensorflow - - keras - - gffutils==0.10.1 - - pyfaidx==0.6.4 - # - scikit-learn - - intervaltree==2.1.0 - # - sklearn-pandas - - tqdm==4.64.0 - # - mmsplice - - concise>=0.6.6 - # other - - ipykernel==6.13.0 - - protobuf==3.20 \ No newline at end of file + # - gffutils==0.10.1 + # - pyfaidx==0.6.4 + # # - scikit-learn + # - intervaltree==2.1.0 + # # - sklearn-pandas + # - tqdm==4.64.0 + # # - mmsplice + # - concise>=0.6.6 + # # other + # - ipykernel==6.13.0 + # - protobuf==3.20 \ No newline at end of file From 4d1a9c1229246cbcda10b207dd40a7ee57d2cd6e Mon Sep 17 00:00:00 2001 From: Haimasree Date: Fri, 17 Jun 2022 11:05:50 +0200 Subject: [PATCH 11/14] added a note why keras is in pip for keras2tf1.yaml --- shared/envs/kipoi-py3-keras2-tf1.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/envs/kipoi-py3-keras2-tf1.yaml b/shared/envs/kipoi-py3-keras2-tf1.yaml index e12fa75c1..886ada9d8 100644 --- a/shared/envs/kipoi-py3-keras2-tf1.yaml +++ b/shared/envs/kipoi-py3-keras2-tf1.yaml @@ -24,7 +24,7 @@ dependencies: - joblib=1.1.0 - tensorflow=1.14 - pip: - - keras==2.1.6 + - keras==2.1.6 # Only FactorNet can not work with keras 2.2.4 wich is the least available option in conda-forge with python=3.7. Keeping keras in pip. - scikit-learn==0.19.2 - gtfparse>=1.0.7 - pyvcf==0.4.3 From 7021c32accd1d53fb86bc311b39399752b676d46 Mon Sep 17 00:00:00 2001 From: Haimasree Date: Fri, 17 Jun 2022 11:42:54 +0200 Subject: [PATCH 12/14] reverting config file --- .circleci/config.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8169b74d0..dbf5fb5a3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -217,14 +217,13 @@ workflows: ignore: - master - test_all - - sharedenvupdate - test_new_models_singularity: filters: branches: ignore: - master - test_all - - sharedenvupdate + test-all-branch: jobs: - test_all_models: @@ -257,7 +256,6 @@ workflows: only: - master - test_all - - sharedenvupdate - sync_containers: context: - sync-kipoi-containers From c05eb6e92aaf41a3ce9bcc804bc960638bb29ad6 Mon Sep 17 00:00:00 2001 From: Haimasree Date: Fri, 17 Jun 2022 11:54:29 +0200 Subject: [PATCH 13/14] cleanup --- shared/envs/kipoi-py3-keras2-tf2.yaml | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/shared/envs/kipoi-py3-keras2-tf2.yaml b/shared/envs/kipoi-py3-keras2-tf2.yaml index 87e411913..d9a9655c9 100644 --- a/shared/envs/kipoi-py3-keras2-tf2.yaml +++ b/shared/envs/kipoi-py3-keras2-tf2.yaml @@ -24,18 +24,5 @@ dependencies: - tensorflow - keras - pip: - # - gtfparse>=1.0.7 - # - pyvcf==0.4.3 - kipoi - - kipoiseq - # - gffutils==0.10.1 - # - pyfaidx==0.6.4 - # # - scikit-learn - # - intervaltree==2.1.0 - # # - sklearn-pandas - # - tqdm==4.64.0 - # # - mmsplice - # - concise>=0.6.6 - # # other - # - ipykernel==6.13.0 - # - protobuf==3.20 \ No newline at end of file + - kipoiseq \ No newline at end of file From 9cb8e639472564e5d952c3e083bc28a83ef80e5a Mon Sep 17 00:00:00 2001 From: Haimasree Date: Fri, 17 Jun 2022 12:00:15 +0200 Subject: [PATCH 14/14] additional note --- shared/envs/kipoi-py3-keras1.2.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/envs/kipoi-py3-keras1.2.yaml b/shared/envs/kipoi-py3-keras1.2.yaml index e424f1e2e..65e2317f1 100644 --- a/shared/envs/kipoi-py3-keras1.2.yaml +++ b/shared/envs/kipoi-py3-keras1.2.yaml @@ -14,7 +14,7 @@ dependencies: - pip=21.3.1 - tensorflow=1.14 - pip: - - keras==1.2.2 + - keras==1.2.2 # keras 1.2.2 is not available with python 3.7 in conda-forge - pyvcf==0.4.3 - kipoi_veff - kipoi_interpret