Merge branch 'master' into coggan_behavior_refactor

.github/workflows/score_new_plugins.yml

@@ -32,10 +32,10 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Set up Python 3.7
+      - name: Set up Python 3.11
         uses: actions/setup-python@v4
         with:
-          python-version: 3.7
+          python-version: 3.11
 
       - name: Configure AWS Credentials
         uses: aws-actions/configure-aws-credentials@v1
@@ -176,10 +176,10 @@ jobs:
       - name: Check out repository code
         uses: actions/checkout@v4
 
-      - name: Set up Python 3.7
+      - name: Set up Python 3.11
         uses: actions/setup-python@v4
         with:
-          python-version: 3.7
+          python-version: 3.11
 
       - name: Build project
         run: |

.github/workflows/sync_develop_with_master.yml

@@ -16,6 +16,7 @@ jobs:
         id: init
         run: |
           echo "Starting branch synchronization of ${{ github.repository }}"
+          
   create_pr_for_nonplugin:
     name: Synchronizing non-plugin PR
     needs: start  # This job now needs the 'start' job to complete first
@@ -34,15 +35,21 @@ jobs:
         run: |
           git fetch origin master
           git reset --hard origin/master
+      - name: Get commit summary
+        id: commit_summary
+        run: |
+          git log -1 --pretty=format:"%s"
+          echo "::set-output name=summary::$(git log -1 --pretty=format:"%s")"
       - name: Create pull request in develop
         uses: peter-evans/create-pull-request@v6
         with:
           token: '${{ secrets.PAT }}'
           commit-message: Sync master into develop
-          title: Sync master into develop
+          title: Sync master into develop. Triggered by PR #${{ github.event.pull_request.number }}
           body: >-
             This PR syncs the latest changes from the master branch into the
             develop branch.
+            Commit Summary: ${{ steps.commit_summary.outputs.summary }}
           base: develop
           branch: 'developer-sync-pr-${{ github.event.pull_request.number }}'
 
@@ -85,27 +92,48 @@ jobs:
             fi
           }
       - name: Push changes to develop (if merge is successful)
-        if: steps.merge.conclusion == 'success'
+        if: steps.merge.outcome == 'success'
         run: | #Use force-with-lease to prevent accidental overwrite if branch has been updated. If fails, rebase the update and retry
           git push origin develop --force-with-lease || {
             echo "Push failed due to updates in develop. Attempting to rebase and retry..."
             git fetch origin develop
             git rebase origin/develop
             git push origin develop --force-with-lease
           }
+      - name: Get commit summary
+        id: commit_summary
+        run: |
+          git log -1 --pretty=format:"%s"
+          echo "::set-output name=summary::$(git log -1 --pretty=format:"%s")"
       - name: Create pull request for merge conflicts
         if: steps.merge.outputs.merge_conflict == 'true'
         uses: peter-evans/create-pull-request@v6
         with:
           token: '${{ secrets.PAT }}'
           commit-message: Merge master into develop with conflict resolution
-          title: Resolve conflicts between master and develop
-          body: This PR resolves merge conflicts between master and develop.
+          title: Resolve conflicts between master and develop. Triggered by PR #${{ github.event.pull_request.number }}
+          body: |
+            This PR resolves merge conflicts between master and develop.
+            Commit Summary: ${{ steps.commit_summary.outputs.summary }}
           base: develop
           branch: 'developer-sync-pr-conflict-${{ github.event.pull_request.number }}'
       - name: Handle other merge failures
         if: failure() && steps.merge.outputs.merge_conflict != 'true'
         run: >
           echo "Handle non-conflict related failure, such as network issues or missing branches"
   
-          # Possibly incorporate additional handling logic here (e.g.,notifications or retries)
+          # Possibly incorporate additional handling logic here (e.g.,notifications or retries)
+
+
+  no_changes:
+    name: "No Changes Made. No synchronization needed."
+    needs: start
+    if: >
+      (
+        needs.create_pr_for_nonplugin.result != 'success' && 
+        needs.auto_sync_for_plugin.result != 'success'
+      )
+    runs-on: ubuntu-latest
+    steps:
+      - name: Echo no changes
+        run: echo "No changes were made to master branch 👍"

.readthedocs.yml

@@ -3,7 +3,7 @@ version: 2
 build:
   os: "ubuntu-20.04"
   tools:
-   python: "3.7"
+   python: "3.11"
 
 python:
    install:

.travis.yml

@@ -1,5 +1,6 @@
 version: ~> 1.0
 language: python
+dist: jammy
 env:
   global:
     - PYTEST_SETTINGS="not requires_gpu and not memory_intense and not slow and not travis_slow"
@@ -9,7 +10,7 @@ env:
     - WEB_SUBMISSION="False"
 before_install:
 - pip install --upgrade pip
-- pip install setuptools==60.5.0
+- pip install setuptools
 - pip install pytest
 # download large files
 - pip install awscli
@@ -31,18 +32,18 @@ import:
 
 jobs:
   include:
-    - name: 3.7 public
-      python: '3.7.13'
-    - name: 3.7 private
+    - name: 3.11 public
+      python: '3.11'
+    - name: 3.11 private
       if: fork = false
-      python: '3.7.13'
+      python: '3.11'
       env:
       - PRIVATE_ACCESS=1
       - secure: f1rWEwrslh7qa2g/QlKs001sGC3uaOxZNQSfNOPj+TMCqEo2c6OzImC4hyz+WqCyc6N/lFT4yYo2RhvaqStHMRmu/+9aZmuH05Bb0KQpfzNFA+yGa/U5WR3/4u6KRvDAeNEi9drT2LuacTyGbldmQsquujK0jrPpFWpe7zUUKv0zb0lJf0zcjeSrZlDXLlgD6DCqow7OqHRvW04dPZVy1OArRwtPV6DJ6Rqo1MqFQGHJ806VPlXhSoydb7a58dhGajqPjomdmZjhd3wS6Lv6uetTE/VVb4EP4e7n0qfZIx/TpnWG0SR44pcP7OCNARWYANsAivzxnQ0shyXnIzOo8ZcPYiPpt/5D53i5idTBxXyuDaHGQvgwuY5XLZzznEedBgZa4OvjxAXlLEQjdVDfSsZeYaV9gyFkeTlLnK1zvWi0US38eF2Qtm3Sx3D/5TtBKK2n38tyK5gg/XvJNycaXvIl7iVcnI2ifpqD1mUWI6C9j9Tk19/XEpWkwaFi91+0LZF1GhjBu8o3G5Np4RIOKXi3TIHkpbMM5mf11T6Bm9LvEMq1h8bgRQigEbeJF8CbUOSVFv+AaXsggGjQhuwdyvy2JZo+tO1nfhi+kW3XrDGPsz1R7Wfqduyn7UUh5OiFymeZwKseYKnwU47KyCqDwrq5Mnx1MlSidnVmPriadR4=
       - secure: WE7FPwy07VzJTKAd2xwZdBhtmh8jk7ojwk4B2rIcBQu0vwUXc1MgO8tBLD7s08lBedBjqZiLZEW31uPMEyWNysouDt16a5gm2d149LR7flI3MOifBtxINfJuC3eOEG65bPgN/bYEsIpLKnu3469d5nxZkK7xsjbWTxHGoUpLvVPsmHY2ZM5/jftybs7fI0do4NMG2XffKfZbiFb447Ao3xeQeEfW6IkJllzgGnlG9FJATFidrbwDNdmzAnvPEnDoKAf7ZvhPV0x9yR5V6P4Ck5hxl8mlPdBa1cRMO8s/1ag1c7YJ3AF9ZlwcwqTiGsT8DHTVRxSz4nFHJTMlrm9j84u7WzLZJBhPgF0UeLN3AQgiAZ3c2TFDvjQWeHVuSPkV5GrKlfhSvR82s9yPEdHQxxwYymBbAr6rJR4NtXTyZX0vg8NRKHssZKLSafs/D/pt9xXspqu8HAHc+mS0lCips79XptSr5BEsioil3D2io3tbzrGugpTeJ7oEA787vKn2Cm4XmhyQ0UBhvwsPZ351l27wZYuNV07o9Ik83hN/w4o2v899QQ/zbX42Iy8ZUCWOPX7MV7+TA7SMxru3qx7HL5hDM8kTetxbLB6Ckr+JOdX8L2Fb5L3TVDpsvfv0ebXgwaQR/ez8/7bcXmBqcERApHDz73HaMXUap+iDR4FLdXE=
       - AWS_DEFAULT_REGION=us-east-1
     - stage: "Automerge check"
-      python: '3.7.13'
+      python: '3.11'
       install:
       - pip install --no-cache-dir torch torchvision --default-timeout=1000 --retries=5
       - pip install --no-cache-dir -e ".[test]"

README.md

@@ -22,9 +22,9 @@ To contribute, please [send in a pull request](https://github.com/brain-score/vi
 
 ## Local installation
 
-You will need Python = 3.7 and pip >= 18.1.
+You will need Python = 3.11 and pip >= 18.1.
 
-`pip install git+https://github.com/brain-score/vision`
+`pip install git+https://github.com/brain-score/vision.git`
 
 Test if the installation is successful by scoring a model on a public benchmark:
 

brainscore_vision/benchmark_helpers/__init__.py

@@ -1,6 +1,7 @@
 from typing import Union
 
 import numpy as np
+import hashlib
 
 from brainio.assemblies import NeuroidAssembly, DataAssembly
 from brainscore_core import Score
@@ -18,6 +19,13 @@ def __init__(self, features: Union[DataAssembly, dict], visual_degrees):
         self.features = features
         self._visual_degrees = visual_degrees
 
+    @property
+    def identifier(self) -> str:
+        # serialize the features to a string and create hash
+        features_data = str(self.features)
+        features_hash = hashlib.md5(features_data.encode('utf-8')).hexdigest()
+        return f"precomputed-{features_hash}"
+
     def visual_degrees(self) -> int:
         return self._visual_degrees
 

brainscore_vision/benchmark_helpers/test_helper.py

@@ -7,6 +7,7 @@
 from brainio.assemblies import NeuroidAssembly, PropertyAssembly
 from brainscore_vision import load_benchmark
 from brainscore_vision.model_interface import BrainModel
+from brainscore_vision.data_helpers import s3
 from . import PrecomputedFeatures
 
 
@@ -68,6 +69,8 @@ def run_test_properties(self, benchmark: str, files: dict, expected: float):
         for current_stimulus in stimulus_identifiers:
             stimulus_set = load_stimulus_set(current_stimulus)
             path = Path(__file__).parent / files[current_stimulus]
+            s3.download_file_if_not_exists(local_path=path,
+                               bucket='brainscore-unittests', remote_filepath=f'tests/test_benchmarks/{files[current_stimulus]}')
             features = PropertyAssembly.from_files(path,
                                                    stimulus_set_identifier=stimulus_set.identifier,
                                                    stimulus_set=stimulus_set)

brainscore_vision/benchmarks/coggan2024_behavior/benchmark.py

@@ -106,9 +106,12 @@ def __call__(self, candidate: BrainModel) -> Score:
             data.model_prediction == data.object_class, dtype=int)
 
         # get correlation between model and human performance across conditions
-        performance = (data[data.visibility < 1]
+        performance = (
+            data[data.visibility < 1]
             .groupby(['subject', 'occluder_type', 'occluder_color'])
-            .mean(['human_accuracy', 'model_accuracy'])).reset_index()
+            .mean(numeric_only=True)
+            .reset_index()
+        )
         scores = performance.groupby('subject').apply(
             lambda df: np.corrcoef(df.human_accuracy, df.model_accuracy)[0, 1])
         score = Score(np.mean(scores))
@@ -131,8 +134,9 @@ def get_noise_ceiling(performance: pd.DataFrame) -> Score:
     nc = []
     for subject in performance.subject.unique():
         performance_ind = performance[performance.subject == subject]
-        performance_grp = (performance[performance.subject != subject]
-            .groupby(['occluder_type', 'occluder_color']).mean())
+        performance_grp = performance[performance.subject != subject]
+        numeric_cols = performance_grp.select_dtypes(include=np.number).columns
+        performance_grp = performance_grp.groupby(['occluder_type', 'occluder_color'])[numeric_cols].mean()
         merged_df = performance_ind.merge(
             performance_grp, on=['occluder_type', 'occluder_color'])
         nc.append(np.corrcoef(merged_df.human_accuracy_x, merged_df.human_accuracy_y)[0, 1])

brainscore_vision/benchmarks/igustibagus2024/domain_transfer_analysis.py

@@ -5,6 +5,8 @@
 import pandas as pd
 from sklearn.linear_model import RidgeClassifierCV
 from sklearn.model_selection import train_test_split
+from sklearn.pipeline import Pipeline
+from sklearn.preprocessing import StandardScaler
 from tqdm import tqdm
 
 # import brain-score specific libraries
@@ -89,7 +91,10 @@ def __call__(self, candidate: BrainModel) -> Score:
 
 def OOD_AnalysisBenchmark():
     return _OOD_AnalysisBenchmark(
-        classifier=RidgeClassifierCV(alphas=[0.0001, 0.001, 0.01, 0.1, 1, 10], fit_intercept=True, normalize=True)
+        classifier=Pipeline([
+            ('scaler', StandardScaler()), 
+            ('classifier', RidgeClassifierCV(alphas=[0.0001, 0.001, 0.01, 0.1, 1, 10], fit_intercept=True))
+        ])
     )
 
 

brainscore_vision/benchmarks/kar2019/test.py

@@ -24,7 +24,7 @@ def test_Kar2019ost_cornet_s():
     filename = 'cornet_s-kar2019.nc'
     filepath = Path(__file__).parent / filename
     s3.download_file_if_not_exists(local_path=filepath,
-                                   bucket='brainio-brainscore', remote_filepath=f'tests/test_benchmarks/{filename}')
+                                   bucket='brainscore-unittests', remote_filepath=f'tests/test_benchmarks/{filename}')
     precomputed_features = NeuroidAssembly.from_files(
         filepath,
         stimulus_set_identifier=benchmark._assembly.stimulus_set.identifier,

brainscore_vision/benchmarks/lonnqvist2024/__init__.py

@@ -0,0 +1,8 @@
+from brainscore_vision import benchmark_registry
+from . import benchmark
+
+benchmark_registry['Lonnqvist2024_InlabInstructionsBehavioralAccuracyDistance'] = lambda: benchmark._Lonnqvist2024BehavioralAccuracyDistanceInlabInstructions()
+benchmark_registry['Lonnqvist2024_InlabNoInstructionsBehavioralAccuracyDistance'] = lambda: benchmark._Lonnqvist2024BehavioralAccuracyDistanceInlabNoInstructions()
+benchmark_registry['Lonnqvist2024_OnlineNoInstructionsBehavioralAccuracyDistance'] = lambda: benchmark._Lonnqvist2024BehavioralAccuracyDistanceOnlineNoInstructions()
+
+benchmark_registry['Lonnqvist2024_EngineeringAccuracy'] = lambda: benchmark._Lonnqvist2024EngineeringAccuracy()