From 157dbe31691830ad3f47e0899f067b0a99e2341d Mon Sep 17 00:00:00 2001
From: Martin Schrimpf <m4rtinsch@gmail.com>
Date: Fri, 21 Feb 2020 17:25:28 -0500
Subject: [PATCH] add Kar2018{hvm,cocogray} benchmarks

not all unit tests pass. Precomputed features don't work yet because the StimulusSet differs (see https://github.com/brain-score/brainio_contrib/pull/24), and the later numbers have not yet been run
---
 brainscore/benchmarks/kar2018.py       | 41 ++++++++++++++++++++++++++
 tests/test_benchmarks/test___init__.py | 18 +++++++++++
 2 files changed, 59 insertions(+)
 create mode 100644 brainscore/benchmarks/kar2018.py

diff --git a/brainscore/benchmarks/kar2018.py b/brainscore/benchmarks/kar2018.py
new file mode 100644
index 000000000..00ecd4d18
--- /dev/null
+++ b/brainscore/benchmarks/kar2018.py
@@ -0,0 +1,41 @@
+import brainscore
+from brainscore.benchmarks._neural_common import NeuralBenchmark, average_repetition
+from brainscore.metrics.ceiling import InternalConsistency
+from brainscore.metrics.regression import CrossRegressedCorrelation, pls_regression, pearsonr_correlation
+from brainscore.utils import LazyLoad
+
+
+def DicarloKar2018hvmPLS():
+    assembly_repetition = LazyLoad(lambda: load_assembly(stimuli='hvm', average_repetitions=False))
+    assembly = LazyLoad(lambda: load_assembly(stimuli='hvm', average_repetitions=True))
+    similarity_metric = CrossRegressedCorrelation(
+        regression=pls_regression(), correlation=pearsonr_correlation(),
+        crossvalidation_kwargs=dict(stratification_coord='object_name'))
+    ceiler = InternalConsistency()
+    return NeuralBenchmark(identifier=f'dicarlo.Kar2018hvm-pls', version=1,
+                           assembly=assembly, similarity_metric=similarity_metric,
+                           ceiling_func=lambda: ceiler(assembly_repetition),
+                           parent='IT', paper_link=None)
+
+
+def DicarloKar2018cocoPLS():
+    assembly_repetition = LazyLoad(lambda: load_assembly(stimuli='cocogray', average_repetitions=False))
+    assembly = LazyLoad(lambda: load_assembly(stimuli='cocogray', average_repetitions=True))
+    similarity_metric = CrossRegressedCorrelation(
+        regression=pls_regression(), correlation=pearsonr_correlation(),
+        crossvalidation_kwargs=dict(stratification_coord='label'))
+    ceiler = InternalConsistency()
+    return NeuralBenchmark(identifier=f'dicarlo.Kar2018coco-pls', version=1,
+                           assembly=assembly, similarity_metric=similarity_metric,
+                           ceiling_func=lambda: ceiler(assembly_repetition),
+                           parent='IT', paper_link=None)
+
+
+def load_assembly(stimuli, average_repetitions):
+    assembly = brainscore.get_assembly(name=f'dicarlo.Kar2018{stimuli}')
+    assembly = assembly.squeeze("time_bin")
+    assembly.load()
+    assembly = assembly.transpose('presentation', 'neuroid')
+    if average_repetitions:
+        assembly = average_repetition(assembly)
+    return assembly
diff --git a/tests/test_benchmarks/test___init__.py b/tests/test_benchmarks/test___init__.py
index 358567b0c..c58d3bd9f 100644
--- a/tests/test_benchmarks/test___init__.py
+++ b/tests/test_benchmarks/test___init__.py
@@ -28,6 +28,10 @@ class TestStandardized:
                      marks=pytest.mark.memory_intense),
         pytest.param('dicarlo.Majaj2015.IT-rdm', approx(.887618, abs=.001),
                      marks=pytest.mark.memory_intense),
+        pytest.param('dicarlo.Kar2018hvm-pls', approx(.842015, abs=.001),
+                     marks=pytest.mark.memory_intense),
+        pytest.param('dicarlo.Kar2018coco-pls', approx(.795665, abs=.001),
+                     marks=pytest.mark.memory_intense),
     ])
     def test_ceilings(self, benchmark, expected):
         benchmark = benchmark_pool[benchmark]
@@ -45,6 +49,10 @@ def test_ceilings(self, benchmark, expected):
                      marks=pytest.mark.memory_intense),
         pytest.param('dicarlo.Majaj2015.IT-pls', approx(.823433, abs=.001),
                      marks=pytest.mark.memory_intense),
+        pytest.param('dicarlo.Kar2018hvm-pls', approx(.823433, abs=.001),
+                     marks=pytest.mark.memory_intense),
+        pytest.param('dicarlo.Kar2018coco-pls', approx(.856989, abs=.001),
+                     marks=pytest.mark.memory_intense),
     ])
     def test_self_regression(self, benchmark, expected):
         benchmark = benchmark_pool[benchmark]
@@ -94,6 +102,16 @@ def test_FreemanZiemba2013(self, benchmark, expected):
     def test_Majaj2015(self, benchmark, expected):
         self.run_test(benchmark=benchmark, file='alexnet-majaj2015.private-features.12.pkl', expected=expected)
 
+    @pytest.mark.memory_intense
+    def test_Kar2018hvm(self):
+        self.run_test(benchmark='dicarlo.Kar2018hvm-pls', file='alexnet-hvm-features.12.pkl',
+                      expected=approx(.490236, abs=.005))
+
+    @pytest.mark.memory_intense
+    def test_Kar2018coco(self):
+        self.run_test(benchmark='dicarlo.Kar2018coco-pls', file='alexnet-cocogray-features.12.pkl',
+                      expected=approx(.490236, abs=.005))
+
     @pytest.mark.memory_intense
     @pytest.mark.requires_gpu
     def test_IT_mask_alexnet(self):