Skip to content

Commit

Permalink
Merge branch 'master' into ep/add_resnet-50_v2
Browse files Browse the repository at this point in the history
KartikP authored Sep 25, 2024

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature.
2 parents 1102c40 + e76e3e3 commit cea88e9
Showing 140 changed files with 2,672 additions and 1,049 deletions.
8 changes: 4 additions & 4 deletions .github/workflows/score_new_plugins.yml
Original file line number Diff line number Diff line change
@@ -32,10 +32,10 @@ jobs:
with:
fetch-depth: 0

- name: Set up Python 3.7
- name: Set up Python 3.11
uses: actions/setup-python@v4
with:
python-version: 3.7
python-version: 3.11

- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v1
@@ -176,10 +176,10 @@ jobs:
- name: Check out repository code
uses: actions/checkout@v4

- name: Set up Python 3.7
- name: Set up Python 3.11
uses: actions/setup-python@v4
with:
python-version: 3.7
python-version: 3.11

- name: Build project
run: |
139 changes: 139 additions & 0 deletions .github/workflows/sync_develop_with_master.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,139 @@
name: Sync develop with master
'on':
pull_request:
types:
- closed
branches:
- master
permissions:
contents: write
jobs:
start:
name: "Starting -🤞"
runs-on: ubuntu-latest
steps:
- name: Starting
id: init
run: |
echo "Starting branch synchronization of ${{ github.repository }}"
create_pr_for_nonplugin:
name: Synchronizing non-plugin PR
needs: start # This job now needs the 'start' job to complete first
if: >
github.event.pull_request.merged == true &&
!(startsWith(github.event.pull_request.head.ref, 'web_submission_') &&
contains(github.event.pull_request.title, 'brain-score.org submission'))
runs-on: ubuntu-latest
steps:
- name: Check out the develop branch
uses: actions/checkout@v3
with:
fetch-depth: 0 # Fetch full history
ref: develop
- name: Reset the develop branch
run: |
git fetch origin master
git reset --hard origin/master
- name: Get commit summary
id: commit_summary
run: |
git log -1 --pretty=format:"%s"
echo "::set-output name=summary::$(git log -1 --pretty=format:"%s")"
- name: Create pull request in develop
uses: peter-evans/create-pull-request@v6
with:
token: '${{ secrets.PAT }}'
commit-message: Sync master into develop
title: Sync master into develop. Triggered by PR #${{ github.event.pull_request.number }}
body: >-
This PR syncs the latest changes from the master branch into the
develop branch.
Commit Summary: ${{ steps.commit_summary.outputs.summary }}
base: develop
branch: 'developer-sync-pr-${{ github.event.pull_request.number }}'

auto_sync_for_plugin:
needs: start
if: >
github.event.pull_request.merged == true &&
startsWith(github.event.pull_request.head.ref, 'web_submission_') &&
contains(github.event.pull_request.title, 'brain-score.org submission')
runs-on: ubuntu-latest
steps:
- name: Checkout the develop branch
uses: actions/checkout@v3
with:
fetch-depth: 0
ref: develop
- name: Configure Git user
run: |
git config --global user.name "Branch Synchronizer"
git config --global user.email "[email protected]"
- name: Ensure develop branch is updated
run: |
git fetch origin develop
git checkout develop
git merge origin/develop
# Fetch latest change from master, checkout develop, merge changes from master to develop.
# Includes conflict handling
- name: Merge master into develop
id: merge
run: |
git fetch origin master
git checkout develop
git merge origin/master || {
if git diff --name-only --diff-filter=U | grep -q '.'; then
echo "Merge conflict detected"
echo "::set-output name=merge_conflict::true"
else
echo "Merge failed due to another reason"
exit 1
fi
}
- name: Push changes to develop (if merge is successful)
if: steps.merge.outcome == 'success'
run: | #Use force-with-lease to prevent accidental overwrite if branch has been updated. If fails, rebase the update and retry
git push origin develop --force-with-lease || {
echo "Push failed due to updates in develop. Attempting to rebase and retry..."
git fetch origin develop
git rebase origin/develop
git push origin develop --force-with-lease
}
- name: Get commit summary
id: commit_summary
run: |
git log -1 --pretty=format:"%s"
echo "::set-output name=summary::$(git log -1 --pretty=format:"%s")"
- name: Create pull request for merge conflicts
if: steps.merge.outputs.merge_conflict == 'true'
uses: peter-evans/create-pull-request@v6
with:
token: '${{ secrets.PAT }}'
commit-message: Merge master into develop with conflict resolution
title: Resolve conflicts between master and develop. Triggered by PR #${{ github.event.pull_request.number }}
body: |
This PR resolves merge conflicts between master and develop.
Commit Summary: ${{ steps.commit_summary.outputs.summary }}
base: develop
branch: 'developer-sync-pr-conflict-${{ github.event.pull_request.number }}'
- name: Handle other merge failures
if: failure() && steps.merge.outputs.merge_conflict != 'true'
run: >
echo "Handle non-conflict related failure, such as network issues or missing branches"
# Possibly incorporate additional handling logic here (e.g.,notifications or retries)


no_changes:
name: "No Changes Made. No synchronization needed."
needs: start
if: >
(
needs.create_pr_for_nonplugin.result != 'success' &&
needs.auto_sync_for_plugin.result != 'success'
)
runs-on: ubuntu-latest
steps:
- name: Echo no changes
run: echo "No changes were made to master branch 👍"
2 changes: 1 addition & 1 deletion .readthedocs.yml
Original file line number Diff line number Diff line change
@@ -3,7 +3,7 @@ version: 2
build:
os: "ubuntu-20.04"
tools:
python: "3.7"
python: "3.11"

python:
install:
13 changes: 7 additions & 6 deletions .travis.yml
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
version: ~> 1.0
language: python
dist: jammy
env:
global:
- PYTEST_SETTINGS="not requires_gpu and not memory_intense and not slow and not travis_slow"
@@ -9,7 +10,7 @@ env:
- WEB_SUBMISSION="False"
before_install:
- pip install --upgrade pip
- pip install setuptools==60.5.0
- pip install setuptools
- pip install pytest
# download large files
- pip install awscli
@@ -31,18 +32,18 @@ import:

jobs:
include:
- name: 3.7 public
python: '3.7.13'
- name: 3.7 private
- name: 3.11 public
python: '3.11'
- name: 3.11 private
if: fork = false
python: '3.7.13'
python: '3.11'
env:
- PRIVATE_ACCESS=1
- secure: f1rWEwrslh7qa2g/QlKs001sGC3uaOxZNQSfNOPj+TMCqEo2c6OzImC4hyz+WqCyc6N/lFT4yYo2RhvaqStHMRmu/+9aZmuH05Bb0KQpfzNFA+yGa/U5WR3/4u6KRvDAeNEi9drT2LuacTyGbldmQsquujK0jrPpFWpe7zUUKv0zb0lJf0zcjeSrZlDXLlgD6DCqow7OqHRvW04dPZVy1OArRwtPV6DJ6Rqo1MqFQGHJ806VPlXhSoydb7a58dhGajqPjomdmZjhd3wS6Lv6uetTE/VVb4EP4e7n0qfZIx/TpnWG0SR44pcP7OCNARWYANsAivzxnQ0shyXnIzOo8ZcPYiPpt/5D53i5idTBxXyuDaHGQvgwuY5XLZzznEedBgZa4OvjxAXlLEQjdVDfSsZeYaV9gyFkeTlLnK1zvWi0US38eF2Qtm3Sx3D/5TtBKK2n38tyK5gg/XvJNycaXvIl7iVcnI2ifpqD1mUWI6C9j9Tk19/XEpWkwaFi91+0LZF1GhjBu8o3G5Np4RIOKXi3TIHkpbMM5mf11T6Bm9LvEMq1h8bgRQigEbeJF8CbUOSVFv+AaXsggGjQhuwdyvy2JZo+tO1nfhi+kW3XrDGPsz1R7Wfqduyn7UUh5OiFymeZwKseYKnwU47KyCqDwrq5Mnx1MlSidnVmPriadR4=
- secure: WE7FPwy07VzJTKAd2xwZdBhtmh8jk7ojwk4B2rIcBQu0vwUXc1MgO8tBLD7s08lBedBjqZiLZEW31uPMEyWNysouDt16a5gm2d149LR7flI3MOifBtxINfJuC3eOEG65bPgN/bYEsIpLKnu3469d5nxZkK7xsjbWTxHGoUpLvVPsmHY2ZM5/jftybs7fI0do4NMG2XffKfZbiFb447Ao3xeQeEfW6IkJllzgGnlG9FJATFidrbwDNdmzAnvPEnDoKAf7ZvhPV0x9yR5V6P4Ck5hxl8mlPdBa1cRMO8s/1ag1c7YJ3AF9ZlwcwqTiGsT8DHTVRxSz4nFHJTMlrm9j84u7WzLZJBhPgF0UeLN3AQgiAZ3c2TFDvjQWeHVuSPkV5GrKlfhSvR82s9yPEdHQxxwYymBbAr6rJR4NtXTyZX0vg8NRKHssZKLSafs/D/pt9xXspqu8HAHc+mS0lCips79XptSr5BEsioil3D2io3tbzrGugpTeJ7oEA787vKn2Cm4XmhyQ0UBhvwsPZ351l27wZYuNV07o9Ik83hN/w4o2v899QQ/zbX42Iy8ZUCWOPX7MV7+TA7SMxru3qx7HL5hDM8kTetxbLB6Ckr+JOdX8L2Fb5L3TVDpsvfv0ebXgwaQR/ez8/7bcXmBqcERApHDz73HaMXUap+iDR4FLdXE=
- AWS_DEFAULT_REGION=us-east-1
- stage: "Automerge check"
python: '3.7.13'
python: '3.11'
install:
- pip install --no-cache-dir torch torchvision --default-timeout=1000 --retries=5
- pip install --no-cache-dir -e ".[test]"
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
@@ -22,7 +22,7 @@ To contribute, please [send in a pull request](https://github.com/brain-score/vi

## Local installation

You will need Python = 3.7 and pip >= 18.1.
You will need Python = 3.11 and pip >= 18.1.

`pip install git+https://github.com/brain-score/vision`

8 changes: 8 additions & 0 deletions brainscore_vision/benchmark_helpers/__init__.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
from typing import Union

import numpy as np
import hashlib

from brainio.assemblies import NeuroidAssembly, DataAssembly
from brainscore_core import Score
@@ -18,6 +19,13 @@ def __init__(self, features: Union[DataAssembly, dict], visual_degrees):
self.features = features
self._visual_degrees = visual_degrees

@property
def identifier(self) -> str:
# serialize the features to a string and create hash
features_data = str(self.features)
features_hash = hashlib.md5(features_data.encode('utf-8')).hexdigest()
return f"precomputed-{features_hash}"

def visual_degrees(self) -> int:
return self._visual_degrees

3 changes: 3 additions & 0 deletions brainscore_vision/benchmark_helpers/test_helper.py
Original file line number Diff line number Diff line change
@@ -7,6 +7,7 @@
from brainio.assemblies import NeuroidAssembly, PropertyAssembly
from brainscore_vision import load_benchmark
from brainscore_vision.model_interface import BrainModel
from brainscore_vision.data_helpers import s3
from . import PrecomputedFeatures


@@ -68,6 +69,8 @@ def run_test_properties(self, benchmark: str, files: dict, expected: float):
for current_stimulus in stimulus_identifiers:
stimulus_set = load_stimulus_set(current_stimulus)
path = Path(__file__).parent / files[current_stimulus]
s3.download_file_if_not_exists(local_path=path,
bucket='brainscore-unittests', remote_filepath=f'tests/test_benchmarks/{files[current_stimulus]}')
features = PropertyAssembly.from_files(path,
stimulus_set_identifier=stimulus_set.identifier,
stimulus_set=stimulus_set)
12 changes: 8 additions & 4 deletions brainscore_vision/benchmarks/coggan2024_behavior/benchmark.py
Original file line number Diff line number Diff line change
@@ -75,9 +75,12 @@ def __call__(self, candidate: BrainModel) -> Score:
data.model_prediction == data.object_class, dtype=int)

# get correlation between model and human performance across conditions
performance = (data[data.visibility < 1]
performance = (
data[data.visibility < 1]
.groupby(['subject', 'occluder_type', 'occluder_color'])
.mean(['human_accuracy', 'model_accuracy'])).reset_index()
.mean(numeric_only=True)
.reset_index()
)
scores = performance.groupby('subject').apply(
lambda df: np.corrcoef(df.human_accuracy, df.model_accuracy)[0, 1])
score = Score(np.mean(scores))
@@ -100,8 +103,9 @@ def get_noise_ceiling(performance: pd.DataFrame) -> Score:
nc = []
for subject in performance.subject.unique():
performance_ind = performance[performance.subject == subject]
performance_grp = (performance[performance.subject != subject]
.groupby(['occluder_type', 'occluder_color']).mean())
performance_grp = performance[performance.subject != subject]
numeric_cols = performance_grp.select_dtypes(include=np.number).columns
performance_grp = performance_grp.groupby(['occluder_type', 'occluder_color'])[numeric_cols].mean()
merged_df = performance_ind.merge(
performance_grp, on=['occluder_type', 'occluder_color'])
nc.append(np.corrcoef(merged_df.human_accuracy_x, merged_df.human_accuracy_y)[0, 1])
Original file line number Diff line number Diff line change
@@ -5,6 +5,8 @@
import pandas as pd
from sklearn.linear_model import RidgeClassifierCV
from sklearn.model_selection import train_test_split
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
from tqdm import tqdm

# import brain-score specific libraries
@@ -89,7 +91,10 @@ def __call__(self, candidate: BrainModel) -> Score:

def OOD_AnalysisBenchmark():
return _OOD_AnalysisBenchmark(
classifier=RidgeClassifierCV(alphas=[0.0001, 0.001, 0.01, 0.1, 1, 10], fit_intercept=True, normalize=True)
classifier=Pipeline([
('scaler', StandardScaler()),
('classifier', RidgeClassifierCV(alphas=[0.0001, 0.001, 0.01, 0.1, 1, 10], fit_intercept=True))
])
)


2 changes: 1 addition & 1 deletion brainscore_vision/benchmarks/kar2019/test.py
Original file line number Diff line number Diff line change
@@ -24,7 +24,7 @@ def test_Kar2019ost_cornet_s():
filename = 'cornet_s-kar2019.nc'
filepath = Path(__file__).parent / filename
s3.download_file_if_not_exists(local_path=filepath,
bucket='brainio-brainscore', remote_filepath=f'tests/test_benchmarks/{filename}')
bucket='brainscore-unittests', remote_filepath=f'tests/test_benchmarks/{filename}')
precomputed_features = NeuroidAssembly.from_files(
filepath,
stimulus_set_identifier=benchmark._assembly.stimulus_set.identifier,
5 changes: 5 additions & 0 deletions brainscore_vision/benchmarks/majajhong2015/__init__.py
Original file line number Diff line number Diff line change
@@ -11,3 +11,8 @@

benchmark_registry['MajajHong2015public.V4-pls'] = MajajHongV4PublicBenchmark
benchmark_registry['MajajHong2015public.IT-pls'] = MajajHongITPublicBenchmark

# temporal
from .benchmark import MajajHongV4TemporalPublicBenchmark, MajajHongITTemporalPublicBenchmark
benchmark_registry['MajajHong2015public.V4-temporal-pls'] = lambda: MajajHongV4TemporalPublicBenchmark(time_interval=10)
benchmark_registry['MajajHong2015public.IT-temporal-pls'] = lambda: MajajHongITTemporalPublicBenchmark(time_interval=10)
44 changes: 34 additions & 10 deletions brainscore_vision/benchmarks/majajhong2015/benchmark.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
from brainscore_core import Metric

from brainscore_vision import load_metric, Ceiling, load_ceiling, load_dataset
from brainscore_vision.benchmark_helpers.neural_common import NeuralBenchmark, average_repetition
from brainscore_vision.benchmark_helpers.neural_common import NeuralBenchmark, average_repetition, apply_keep_attrs
from brainscore_vision.model_helpers.brain_transformation.temporal import assembly_time_align

VISUAL_DEGREES = 8
NUMBER_OF_TRIALS = 50
@@ -20,13 +21,14 @@
eprint = {https://www.jneurosci.org/content/35/39/13402.full.pdf},
journal = {Journal of Neuroscience}}"""

pls_metric = lambda: load_metric('pls', crossvalidation_kwargs=dict(stratification_coord='object_name'))

crossvalidation_kwargs = dict(stratification_coord='object_name')
pls_metric = lambda: load_metric('pls', crossvalidation_kwargs=crossvalidation_kwargs)
spantime_pls_metric = lambda: load_metric('spantime_pls', crossvalidation_kwargs=crossvalidation_kwargs)

def _DicarloMajajHong2015Region(region: str, access: str, identifier_metric_suffix: str,
similarity_metric: Metric, ceiler: Ceiling):
assembly_repetition = load_assembly(average_repetitions=False, region=region, access=access)
assembly = load_assembly(average_repetitions=True, region=region, access=access)
similarity_metric: Metric, ceiler: Ceiling, time_interval: float = None):
assembly_repetition = load_assembly(average_repetitions=False, region=region, access=access, time_interval=time_interval)
assembly = load_assembly(average_repetitions=True, region=region, access=access, time_interval=time_interval)
benchmark_identifier = f'MajajHong2015.{region}' + ('.public' if access == 'public' else '')
return NeuralBenchmark(identifier=f'{benchmark_identifier}-{identifier_metric_suffix}', version=3,
assembly=assembly, similarity_metric=similarity_metric,
@@ -60,13 +62,35 @@ def MajajHongITPublicBenchmark():
ceiler=load_ceiling('internal_consistency'))


def load_assembly(average_repetitions, region, access='private'):
assembly = load_dataset(f'MajajHong2015.{access}')
def MajajHongV4TemporalPublicBenchmark(time_interval: float = None):
return _DicarloMajajHong2015Region(region='V4', access='public', identifier_metric_suffix='pls',
similarity_metric=spantime_pls_metric(), time_interval=time_interval,
ceiler=load_ceiling('internal_consistency_temporal'))


def MajajHongITTemporalPublicBenchmark(time_interval: float = None):
return _DicarloMajajHong2015Region(region='IT', access='public', identifier_metric_suffix='pls',
similarity_metric=spantime_pls_metric(), time_interval=time_interval,
ceiler=load_ceiling('internal_consistency_temporal'))


def load_assembly(average_repetitions: bool, region: str, access: str = 'private', time_interval: float = None):
temporal = time_interval is not None
if not temporal:
assembly = load_dataset(f'MajajHong2015.{access}')
assembly = assembly.squeeze("time_bin")
else:
assembly = load_dataset(f'MajajHong2015.temporal.{access}')
assembly = assembly.__class__(assembly)
target_time_bins = [
(t, t+time_interval) for t in range(0, assembly.time_bin_end.max().item()-time_interval, time_interval)
]
assembly = apply_keep_attrs(assembly, lambda assembly: assembly_time_align(assembly, target_time_bins))

assembly = assembly.sel(region=region)
assembly['region'] = 'neuroid', [region] * len(assembly['neuroid'])
assembly = assembly.squeeze("time_bin")
assembly.load()
assembly = assembly.transpose('presentation', 'neuroid')
assembly = assembly.transpose('presentation', 'neuroid', ...)
if average_repetitions:
assembly = average_repetition(assembly)
return assembly
Loading

0 comments on commit cea88e9

Please sign in to comment.