Merge branch 'master' into sw/bs_version_update

.travis.yml

@@ -14,8 +14,7 @@ before_install:
 # download large files
 - pip install awscli
 install:
-- travis_wait pip install --no-cache-dir torch
-- travis_wait pip install --no-cache-dir torchvision
+- pip install --no-cache-dir torch torchvision --default-timeout=1000 --retries=5
 - pip install --no-cache-dir -e ".[test]"
 # install conda for plugin runner
 - wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh
@@ -44,13 +43,12 @@ jobs:
     - stage: "Automerge check"
       python: '3.11'
       install:
-      - travis_wait pip install --no-cache-dir torch
-      - travis_wait pip install --no-cache-dir torchvision
+      - pip install --no-cache-dir torch torchvision --default-timeout=1000 --retries=5
       - pip install --no-cache-dir -e ".[test]"
       if: type = pull_request
       script:
         - |
-          CHANGED_FILES=$( git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" && git fetch && echo $(git diff --name-only origin/$TRAVIS_PULL_REQUEST_BRANCH origin/$TRAVIS_BRANCH -C $TRAVIS_BUILD_DIR) | tr '\n' ' ' ) &&
+          CHANGED_FILES=$( git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" && git fetch && echo $(git diff --name-only origin/$TRAVIS_BRANCH...origin/$TRAVIS_PULL_REQUEST_BRANCH -C $TRAVIS_BUILD_DIR) | tr '\n' ' ' ) &&
           PLUGIN_ONLY=$( python -c "from brainscore_core.plugin_management.parse_plugin_changes import is_plugin_only; is_plugin_only(\"${CHANGED_FILES}\", \"brainscore_${DOMAIN}\")" )
           if [ "$PLUGIN_ONLY" = "True" ] && [ "$TRAVIS_SECURE_ENV_VARS" = true ]; then
             bash ${TRAVIS_BUILD_DIR}/.github/workflows/workflow_trigger.sh $GH_WORKFLOW_TRIGGER $TRAVIS_PULL_REQUEST_SHA "Successful Travis PR build for plugin-only PR" "continuous-integration/travis"; 

brainscore_vision/benchmark_helpers/test_helper.py

@@ -61,8 +61,8 @@ def run_test_properties(self, benchmark: str, files: dict, expected: float):
         benchmark = load_benchmark(benchmark)
         from brainscore_vision import load_stimulus_set
 
-        stimulus_identifiers = np.unique(np.array(['dicarlo.Marques2020_blank', 'dicarlo.Marques2020_receptive_field',
-                                                   'dicarlo.Marques2020_orientation',
+        stimulus_identifiers = np.unique(np.array(['Marques2020_blank', 'Marques2020_receptive_field',
+                                                   'Marques2020_orientation',
                                                    benchmark._assembly.stimulus_set.identifier]))
         precomputed_features = {}
         for current_stimulus in stimulus_identifiers:

brainscore_vision/benchmarks/cadena2017/test.py

@@ -15,7 +15,7 @@ def test(self):
         loader = AssemblyLoader()
         assembly = loader()
         check_standard_format(assembly)
-        assert assembly.attrs['stimulus_set_identifier'] == 'tolias.Cadena2017'
+        assert assembly.attrs['stimulus_set_identifier'] == 'Cadena2017'
         assert len(assembly['presentation']) == 6249
         assert len(assembly['neuroid']) == 166
 

brainscore_vision/benchmarks/imagenet_c/benchmark.py

@@ -75,7 +75,7 @@ class Imagenet_C_Category(BenchmarkBase):
 
     def __init__(self, noise_category, sampling_factor=10):
         self.noise_category = noise_category
-        self.stimulus_set_name = f'dietterich.Hendrycks2019.{noise_category}'
+        self.stimulus_set_name = f'imagenet_c.{noise_category}'
 
         self.sampling_factor = sampling_factor
         self.stimulus_set = self.load_stimulus_set()
@@ -98,11 +98,11 @@ def load_stimulus_set(self):
             _logger.debug(f'Loading local Imagenet-C {self.noise_category}')
             category_path = os.path.join(
                 LOCAL_STIMULUS_DIRECTORY,
-                f'image_dietterich_Hendrycks2019_{self.noise_category}'
+                f'stimulus_imagenet_c_{self.noise_category}'
             )
             loader = SampledStimulusSetLoader(
                 cls=StimulusSet,
-                csv_path=os.path.join(category_path, f'image_dietterich_Hendrycks2019_{self.noise_category}.csv'),
+                csv_path=os.path.join(category_path, f'stimulus_imagenet_c_{self.noise_category}.csv'),
                 stimuli_directory=category_path,
                 sampling_factor=self.sampling_factor
             )

brainscore_vision/benchmarks/marques2020/test.py

@@ -101,7 +101,7 @@ def test_Marques2020(benchmark, expected):
                'Marques2020_orientation': 'alexnet-dicarlo.Marques2020_orientation.nc',
                'Marques2020_spatial_frequency': 'alexnet-dicarlo.Marques2020_spatial_frequency.nc',
                'Marques2020_size': 'alexnet-dicarlo.Marques2020_size.nc',
-               'movshon.FreemanZiemba2013_properties': 'alexnet-movshon.FreemanZiemba2013_properties.nc',
+               'FreemanZiemba2013_properties': 'alexnet-movshon.FreemanZiemba2013_properties.nc',
                },
         expected=expected)
 

brainscore_vision/benchmarks/scialom2024/__init__.py

@@ -0,0 +1,50 @@
+from brainscore_vision import benchmark_registry
+from . import benchmark
+
+# behavioral benchmarks
+benchmark_registry['Scialom2024_rgbBehavioralAccuracyDistance'] = lambda: benchmark._Scialom2024BehavioralAccuracyDistance('rgb')
+benchmark_registry['Scialom2024_contoursBehavioralAccuracyDistance'] = lambda: benchmark._Scialom2024BehavioralAccuracyDistance('contours')
+benchmark_registry['Scialom2024_phosphenes-12BehavioralAccuracyDistance'] = lambda: benchmark._Scialom2024BehavioralAccuracyDistance('phosphenes-12')
+benchmark_registry['Scialom2024_phosphenes-16BehavioralAccuracyDistance'] = lambda: benchmark._Scialom2024BehavioralAccuracyDistance('phosphenes-16')
+benchmark_registry['Scialom2024_phosphenes-21BehavioralAccuracyDistance'] = lambda: benchmark._Scialom2024BehavioralAccuracyDistance('phosphenes-21')
+benchmark_registry['Scialom2024_phosphenes-27BehavioralAccuracyDistance'] = lambda: benchmark._Scialom2024BehavioralAccuracyDistance('phosphenes-27')
+benchmark_registry['Scialom2024_phosphenes-35BehavioralAccuracyDistance'] = lambda: benchmark._Scialom2024BehavioralAccuracyDistance('phosphenes-35')
+benchmark_registry['Scialom2024_phosphenes-46BehavioralAccuracyDistance'] = lambda: benchmark._Scialom2024BehavioralAccuracyDistance('phosphenes-46')
+benchmark_registry['Scialom2024_phosphenes-59BehavioralAccuracyDistance'] = lambda: benchmark._Scialom2024BehavioralAccuracyDistance('phosphenes-59')
+benchmark_registry['Scialom2024_phosphenes-77BehavioralAccuracyDistance'] = lambda: benchmark._Scialom2024BehavioralAccuracyDistance('phosphenes-77')
+benchmark_registry['Scialom2024_phosphenes-100BehavioralAccuracyDistance'] = lambda: benchmark._Scialom2024BehavioralAccuracyDistance('phosphenes-100')
+benchmark_registry['Scialom2024_segments-12BehavioralAccuracyDistance'] = lambda: benchmark._Scialom2024BehavioralAccuracyDistance('segments-12')
+benchmark_registry['Scialom2024_segments-16BehavioralAccuracyDistance'] = lambda: benchmark._Scialom2024BehavioralAccuracyDistance('segments-16')
+benchmark_registry['Scialom2024_segments-21BehavioralAccuracyDistance'] = lambda: benchmark._Scialom2024BehavioralAccuracyDistance('segments-21')
+benchmark_registry['Scialom2024_segments-27BehavioralAccuracyDistance'] = lambda: benchmark._Scialom2024BehavioralAccuracyDistance('segments-27')
+benchmark_registry['Scialom2024_segments-35BehavioralAccuracyDistance'] = lambda: benchmark._Scialom2024BehavioralAccuracyDistance('segments-35')
+benchmark_registry['Scialom2024_segments-46BehavioralAccuracyDistance'] = lambda: benchmark._Scialom2024BehavioralAccuracyDistance('segments-46')
+benchmark_registry['Scialom2024_segments-59BehavioralAccuracyDistance'] = lambda: benchmark._Scialom2024BehavioralAccuracyDistance('segments-59')
+benchmark_registry['Scialom2024_segments-77BehavioralAccuracyDistance'] = lambda: benchmark._Scialom2024BehavioralAccuracyDistance('segments-77')
+benchmark_registry['Scialom2024_segments-100BehavioralAccuracyDistance'] = lambda: benchmark._Scialom2024BehavioralAccuracyDistance('segments-100')
+
+# composites
+benchmark_registry['Scialom2024_phosphenes-allBehavioralErrorConsistency'] = lambda: benchmark._Scialom2024BehavioralErrorConsistency('phosphenes-all')
+benchmark_registry['Scialom2024_segments-allBehavioralErrorConsistency'] = lambda: benchmark._Scialom2024BehavioralErrorConsistency('segments-all')
+
+# engineering benchmarks
+benchmark_registry['Scialom2024_rgbEngineeringAccuracy'] = lambda: benchmark._Scialom2024EngineeringAccuracy('rgb')
+benchmark_registry['Scialom2024_contoursEngineeringAccuracy'] = lambda: benchmark._Scialom2024EngineeringAccuracy('contours')
+benchmark_registry['Scialom2024_phosphenes-12EngineeringAccuracy'] = lambda: benchmark._Scialom2024EngineeringAccuracy('phosphenes-12')
+benchmark_registry['Scialom2024_phosphenes-16EngineeringAccuracy'] = lambda: benchmark._Scialom2024EngineeringAccuracy('phosphenes-16')
+benchmark_registry['Scialom2024_phosphenes-21EngineeringAccuracy'] = lambda: benchmark._Scialom2024EngineeringAccuracy('phosphenes-21')
+benchmark_registry['Scialom2024_phosphenes-27EngineeringAccuracy'] = lambda: benchmark._Scialom2024EngineeringAccuracy('phosphenes-27')
+benchmark_registry['Scialom2024_phosphenes-35EngineeringAccuracy'] = lambda: benchmark._Scialom2024EngineeringAccuracy('phosphenes-35')
+benchmark_registry['Scialom2024_phosphenes-46EngineeringAccuracy'] = lambda: benchmark._Scialom2024EngineeringAccuracy('phosphenes-46')
+benchmark_registry['Scialom2024_phosphenes-59EngineeringAccuracy'] = lambda: benchmark._Scialom2024EngineeringAccuracy('phosphenes-59')
+benchmark_registry['Scialom2024_phosphenes-77EngineeringAccuracy'] = lambda: benchmark._Scialom2024EngineeringAccuracy('phosphenes-77')
+benchmark_registry['Scialom2024_phosphenes-100EngineeringAccuracy'] = lambda: benchmark._Scialom2024EngineeringAccuracy('phosphenes-100')
+benchmark_registry['Scialom2024_segments-12EngineeringAccuracy'] = lambda: benchmark._Scialom2024EngineeringAccuracy('segments-12')
+benchmark_registry['Scialom2024_segments-16EngineeringAccuracy'] = lambda: benchmark._Scialom2024EngineeringAccuracy('segments-16')
+benchmark_registry['Scialom2024_segments-21EngineeringAccuracy'] = lambda: benchmark._Scialom2024EngineeringAccuracy('segments-21')
+benchmark_registry['Scialom2024_segments-27EngineeringAccuracy'] = lambda: benchmark._Scialom2024EngineeringAccuracy('segments-27')
+benchmark_registry['Scialom2024_segments-35EngineeringAccuracy'] = lambda: benchmark._Scialom2024EngineeringAccuracy('segments-35')
+benchmark_registry['Scialom2024_segments-46EngineeringAccuracy'] = lambda: benchmark._Scialom2024EngineeringAccuracy('segments-46')
+benchmark_registry['Scialom2024_segments-59EngineeringAccuracy'] = lambda: benchmark._Scialom2024EngineeringAccuracy('segments-59')
+benchmark_registry['Scialom2024_segments-77EngineeringAccuracy'] = lambda: benchmark._Scialom2024EngineeringAccuracy('segments-77')
+benchmark_registry['Scialom2024_segments-100EngineeringAccuracy'] = lambda: benchmark._Scialom2024EngineeringAccuracy('segments-100')

brainscore_vision/benchmarks/scialom2024/benchmark.py

@@ -0,0 +1,97 @@
+import numpy as np
+
+import brainscore_vision
+from brainio.assemblies import BehavioralAssembly
+from brainscore_vision import load_dataset, load_metric
+from brainscore_vision.benchmark_helpers.screen import place_on_screen
+from brainscore_vision.benchmarks import BenchmarkBase
+from brainscore_vision.metrics import Score
+from brainscore_vision.model_interface import BrainModel
+from brainscore_vision.utils import LazyLoad
+
+BIBTEX = ""  # to appear in a future article
+
+
+class _Scialom2024BehavioralErrorConsistency(BenchmarkBase):
+    def __init__(self, dataset):
+        self._metric = load_metric('error_consistency')
+        self._assembly = LazyLoad(lambda: load_assembly(dataset))
+        self._visual_degrees = 8
+        self._number_of_trials = 1
+
+        super(_Scialom2024BehavioralErrorConsistency, self).__init__(
+            identifier=f'Scialom2024_{dataset}-error_consistency', version=1,
+            ceiling_func=lambda: self._metric.ceiling(self._assembly),
+            parent='Scialom2024',
+            bibtex=BIBTEX)
+
+    def __call__(self, candidate: BrainModel):
+        choice_labels = set(self._assembly['truth'].values)
+        choice_labels = list(sorted(choice_labels))
+        candidate.start_task(BrainModel.Task.label, choice_labels)
+        stimulus_set = place_on_screen(self._assembly.stimulus_set, target_visual_degrees=candidate.visual_degrees(),
+                                       source_visual_degrees=self._visual_degrees)
+        labels = candidate.look_at(stimulus_set, number_of_trials=self._number_of_trials)
+        raw_score = self._metric(labels, self._assembly)
+        ceiling = self.ceiling
+        score = raw_score / ceiling
+        score.attrs['raw'] = raw_score
+        score.attrs['ceiling'] = ceiling
+        return score
+
+
+class _Scialom2024BehavioralAccuracyDistance(BenchmarkBase):
+    # behavioral benchmark
+    def __init__(self, dataset):
+        self._metric = load_metric('accuracy_distance')
+        self._assembly = LazyLoad(lambda: load_assembly(dataset))
+        super(_Scialom2024BehavioralAccuracyDistance, self).__init__(
+            identifier=f'Scialom2024_{dataset}-behavioral_accuracy', version=1,
+            ceiling_func=lambda: self._metric.ceiling(self._assembly),
+            parent='Scialom2024',
+            bibtex=BIBTEX)
+
+    def __call__(self, candidate: BrainModel):
+        choice_labels = set(self._assembly.stimulus_set['truth'].values)
+        choice_labels = list(sorted(choice_labels))
+        candidate.start_task(BrainModel.Task.label, choice_labels)
+        labels = candidate.look_at(self._assembly.stimulus_set, number_of_trials=1)
+        raw_score = self._metric(labels, target=self._assembly)
+        ceiling = self.ceiling
+        score = raw_score / ceiling
+        # ensure score <= 1.0
+        if score.values > 1:
+            score = Score(np.array(1.))
+        score.attrs['raw'] = raw_score
+        score.attrs['ceiling'] = ceiling
+        return score
+
+
+class _Scialom2024EngineeringAccuracy(BenchmarkBase):
+    # engineering/ML benchmark
+    def __init__(self, dataset):
+        self._metric = load_metric('accuracy')
+        # no lazyload because needed in candidate.look_at()
+        self._stimulus_set = load_assembly(dataset).stimulus_set
+        super(_Scialom2024EngineeringAccuracy, self).__init__(
+            identifier=f'Scialom2024_{dataset}-engineering_accuracy', version=1,
+            ceiling_func=lambda: Score(1),
+            parent='Scialom2024-top1',
+            bibtex=BIBTEX)
+
+    def __call__(self, candidate: BrainModel):
+        choice_labels = set(self._stimulus_set['truth'].values)
+        choice_labels = list(sorted(choice_labels))
+        candidate.start_task(BrainModel.Task.label, choice_labels)
+        labels = candidate.look_at(self._stimulus_set, number_of_trials=10)
+        raw_score = self._metric(labels, target=self._stimulus_set['truth'].values)
+        ceiling = Score(np.array(1.))
+        score = raw_score / ceiling
+        score.attrs['raw'] = raw_score
+        score.attrs['ceiling'] = ceiling
+        return score
+
+
+def load_assembly(dataset: str) -> BehavioralAssembly:
+    assembly = brainscore_vision.load_dataset(f'Scialom2024_{dataset}')
+    return assembly