Merge pull request #10 from BeckResearchLab/new_benchmarks

New benchmarks

README.md

@@ -106,8 +106,7 @@ Some of the figures in the manuscript were created during the main pipeline step
 - Figure 2: located at `./analysis/figures/AA_propensities.png` created in notebook `./analysis/probe_model.ipynb`
 - Figure 3: located at `./analysis/figures/disulfide_logits.png` created in notebook `./analysis/probe_model.ipynb`
 - Figure 4: located at `./analysis/figures/estimated_shift_thermo_gen.png` created in notebook `./analysis/dataset_stats.ipynb`
-- Figure 5: located at `./analysis/figures/mAF_scores.png` created in notebook `./analysis/compare_estimated_stability.ipynb`
-- Figure 6: see repo https://zenodo.org/records/10625583
+- Figure 5: see repo https://zenodo.org/records/10625583
 - Figure 7: located at './data/plots/exp_tm_scores.png' created in script `./scripts/zero_shot_experiment.py`
 
 ## License

data/nomelt-model-full/benchmark_results.json

@@ -0,0 +1,20 @@
+{
+  "new": {
+    "lipa": {
+      "pearson_coef": 0.2890140057583261,
+      "pearson_p": 4.670293896734481e-43,
+      "spearman_coef": 0.3205967878373402,
+      "spearman_p": 4.1698546948049733e-53,
+      "frac_qualitative_sig": 0.6286221472483685
+    }
+  },
+  "original": {
+    "lipa": {
+      "pearson_coef": 0.0070886669820527425,
+      "pearson_p": 0.7412644748916395,
+      "spearman_coef": 0.001625281365076926,
+      "spearman_p": 0.9396560014153784,
+      "frac_qualitative_sig": 0.49945479059432124
+    }
+  }
+}

data/nomelt-model-full/fireprotdb_benchmark_results.json

@@ -0,0 +1,12 @@
+{
+  "New": {
+    "spearman_coef": 0.32183325388483064,
+    "spearman_p": 2.11588628138216e-218,
+    "auroc": 0.6122423981479029
+  },
+  "Original": {
+    "spearman_coef": -0.016768034527282,
+    "spearman_p": 0.10965961456982194,
+    "auroc": 0.5013025291347248
+  }
+}