diff --git a/.github/workflows/sync_develop_with_master.yml b/.github/workflows/sync_develop_with_master.yml new file mode 100644 index 000000000..df8cc238d --- /dev/null +++ b/.github/workflows/sync_develop_with_master.yml @@ -0,0 +1,111 @@ +name: Sync develop with master +'on': + pull_request: + types: + - closed + branches: + - master +permissions: + contents: write +jobs: + start: + name: "Starting -🤞" + runs-on: ubuntu-latest + steps: + - name: Starting + id: init + run: | + echo "Starting branch synchronization of ${{ github.repository }}" + create_pr_for_nonplugin: + name: Synchronizing non-plugin PR + needs: start # This job now needs the 'start' job to complete first + if: > + github.event.pull_request.merged == true && + !(startsWith(github.event.pull_request.head.ref, 'web_submission_') && + contains(github.event.pull_request.title, 'brain-score.org submission')) + runs-on: ubuntu-latest + steps: + - name: Check out the develop branch + uses: actions/checkout@v3 + with: + fetch-depth: 0 # Fetch full history + ref: develop + - name: Reset the develop branch + run: | + git fetch origin master + git reset --hard origin/master + - name: Create pull request in develop + uses: peter-evans/create-pull-request@v6 + with: + token: '${{ secrets.PAT }}' + commit-message: Sync master into develop + title: Sync master into develop + body: >- + This PR syncs the latest changes from the master branch into the + develop branch. + base: develop + branch: 'developer-sync-pr-${{ github.event.pull_request.number }}' + + auto_sync_for_plugin: + needs: start + if: > + github.event.pull_request.merged == true && + startsWith(github.event.pull_request.head.ref, 'web_submission_') && + contains(github.event.pull_request.title, 'brain-score.org submission') + runs-on: ubuntu-latest + steps: + - name: Checkout the develop branch + uses: actions/checkout@v3 + with: + fetch-depth: 0 + ref: develop + - name: Configure Git user + run: | + git config --global user.name "Branch Synchronizer" + git config --global user.email "action@github.com" + - name: Ensure develop branch is updated + run: | + git fetch origin develop + git checkout develop + git merge origin/develop + # Fetch latest change from master, checkout develop, merge changes from master to develop. + # Includes conflict handling + - name: Merge master into develop + id: merge + run: | + git fetch origin master + git checkout develop + git merge origin/master || { + if git diff --name-only --diff-filter=U | grep -q '.'; then + echo "Merge conflict detected" + echo "::set-output name=merge_conflict::true" + else + echo "Merge failed due to another reason" + exit 1 + fi + } + - name: Push changes to develop (if merge is successful) + if: steps.merge.conclusion == 'success' + run: | #Use force-with-lease to prevent accidental overwrite if branch has been updated. If fails, rebase the update and retry + git push origin develop --force-with-lease || { + echo "Push failed due to updates in develop. Attempting to rebase and retry..." + git fetch origin develop + git rebase origin/develop + git push origin develop --force-with-lease + } + - name: Create pull request for merge conflicts + if: steps.merge.outputs.merge_conflict == 'true' + uses: peter-evans/create-pull-request@v6 + with: + token: '${{ secrets.PAT }}' + commit-message: Merge master into develop with conflict resolution + title: Resolve conflicts between master and develop + body: This PR resolves merge conflicts between master and develop. + base: develop + branch: 'developer-sync-pr-conflict-${{ github.event.pull_request.number }}' + - name: Handle other merge failures + if: failure() && steps.merge.outputs.merge_conflict != 'true' + run: > + echo "Handle non-conflict related failure, such as network issues or missing branches" + + # Possibly incorporate additional handling logic here (e.g.,notifications or retries) \ No newline at end of file diff --git a/brainscore_vision/models/eBarlow_lmda_02_1000ep/__init__.py b/brainscore_vision/models/eBarlow_lmda_02_1000ep/__init__.py new file mode 100644 index 000000000..f29cd461a --- /dev/null +++ b/brainscore_vision/models/eBarlow_lmda_02_1000ep/__init__.py @@ -0,0 +1,9 @@ +from brainscore_vision import model_registry +from brainscore_vision.model_helpers.brain_transformation import ModelCommitment +from .model import get_model, get_layers + +model_registry["eBarlow_lmda_02_1000ep"] = lambda: ModelCommitment( + identifier="eBarlow_lmda_02_1000ep", + activations_model=get_model("eBarlow_lmda_02_1000ep"), + layers=get_layers("eBarlow_lmda_02_1000ep"), +) diff --git a/brainscore_vision/models/eBarlow_lmda_02_1000ep/model.py b/brainscore_vision/models/eBarlow_lmda_02_1000ep/model.py new file mode 100644 index 000000000..2a4d8797e --- /dev/null +++ b/brainscore_vision/models/eBarlow_lmda_02_1000ep/model.py @@ -0,0 +1,84 @@ +from brainscore_vision.model_helpers.check_submission import check_models +import functools +import os +from urllib.request import urlretrieve +import torchvision.models +from brainscore_vision.model_helpers.activations.pytorch import PytorchWrapper +from brainscore_vision.model_helpers.activations.pytorch import load_preprocess_images +from pathlib import Path +from brainscore_vision.model_helpers import download_weights +import torch +from collections import OrderedDict + +# This is an example implementation for submitting resnet-50 as a pytorch model + +# Attention: It is important, that the wrapper identifier is unique per model! +# The results will otherwise be the same due to brain-scores internal result caching mechanism. +# Please load your pytorch model for usage in CPU. There won't be GPUs available for scoring your model. +# If the model requires a GPU, contact the brain-score team directly. +from brainscore_vision.model_helpers.check_submission import check_models + + +def get_model_list(): + return ["eBarlow_lmda_02_1000ep"] + + +def get_model(name): + assert name == "eBarlow_lmda_02_1000ep" + url = "https://users.flatironinstitute.org/~tyerxa/equi_proj/training_checkpoints/fresh/paired/lmda_0.2/Barlow_1000ep/ep1000-ba625000-rank0" + fh = urlretrieve(url) + state_dict = torch.load(fh[0], map_location=torch.device("cpu"))["state"]["model"] + model = load_composer_classifier(state_dict) + preprocessing = functools.partial(load_preprocess_images, image_size=224) + wrapper = PytorchWrapper(identifier=name, model=model, preprocessing=preprocessing) + wrapper.image_size = 224 + return wrapper + +def load_composer_classifier(sd): + model = torchvision.models.resnet.resnet50() + new_sd = OrderedDict() + for k, v in sd.items(): + if 'lin_cls' in k: + new_sd['fc.' + k.split('.')[-1]] = v + if ".f." not in k: + continue + parts = k.split(".") + idx = parts.index("f") + new_k = ".".join(parts[idx + 1 :]) + new_sd[new_k] = v + model.load_state_dict(new_sd, strict=True) + return model + +def get_layers(name): + assert name == "eBarlow_lmda_02_1000ep" + layers = [ + "layer1.0", + "layer1.1", + "layer1.2", + "layer2.0", + "layer2.1", + "layer2.2", + "layer2.3", + "layer3.0", + "layer3.1", + "layer3.2", + "layer3.3", + "layer3.4", + "layer3.5", + "layer4.0", + "layer4.1", + "layer4.2", + "avgpool", + "fc", + ] + outs = ["conv1", "layer1", "layer2", "layer3", "layer4"] + + return layers + outs + + +def get_bibtex(model_identifier): + return """xx""" + + +if __name__ == "__main__": + check_models.check_base_models(__name__) diff --git a/brainscore_vision/models/eBarlow_lmda_02_1000ep/setup.py b/brainscore_vision/models/eBarlow_lmda_02_1000ep/setup.py new file mode 100644 index 000000000..421914cfb --- /dev/null +++ b/brainscore_vision/models/eBarlow_lmda_02_1000ep/setup.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from setuptools import setup, find_packages + +requirements = [ "torchvision", + "torch" +] + +setup( + packages=find_packages(exclude=['tests']), + include_package_data=True, + install_requires=requirements, + license="MIT license", + zip_safe=False, + keywords='brain-score template', + classifiers=[ + 'Development Status :: 2 - Pre-Alpha', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: MIT License', + 'Natural Language :: English', + 'Programming Language :: Python :: 3.7', + ], + test_suite='tests', +) diff --git a/brainscore_vision/models/eBarlow_lmda_02_1000ep/test.py b/brainscore_vision/models/eBarlow_lmda_02_1000ep/test.py new file mode 100644 index 000000000..e594ba9e1 --- /dev/null +++ b/brainscore_vision/models/eBarlow_lmda_02_1000ep/test.py @@ -0,0 +1 @@ +# Left empty as part of 2023 models migration diff --git a/docs/requirements.txt b/docs/requirements.txt index 0f71f0a9f..23532fd21 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,4 +1,5 @@ -e . Sphinx>=4 +sphinx_rtd_theme recommonmark \ No newline at end of file diff --git a/docs/source/conf.py b/docs/source/conf.py index 295f247a9..1f14b464d 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -17,11 +17,11 @@ # -- Project information ----------------------------------------------------- project = 'Brain-Score' -copyright = '2022, Brain-Score Team' +copyright = '2024, Brain-Score Team' author = 'Brain-Score Team' # The full version, including alpha/beta/rc tags -release = '1.3' +release = '2.0.0' # -- General configuration --------------------------------------------------- @@ -58,7 +58,7 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = 'alabaster' +html_theme = 'sphinx_rtd_theme' # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, diff --git a/docs/source/index.rst b/docs/source/index.rst index c7d629d85..453876e60 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -12,9 +12,8 @@ By following a unified :ref:`interface`, all models can be treated as an experim :caption: Contents: examples - modules/model_tutorial - modules/benchmark_tutorial modules/model_interface + modules/developer_clarifications.rst modules/benchmarks modules/metrics modules/submission diff --git a/docs/source/modules/benchmark_tutorial.rst b/docs/source/modules/benchmark_tutorial.rst deleted file mode 100644 index 258590d79..000000000 --- a/docs/source/modules/benchmark_tutorial.rst +++ /dev/null @@ -1,9 +0,0 @@ -.. _Benchmark_Tutorial: -.. |UnitTestSupport| replace:: We realize that unit tests can be a hurdle and we can take over this task for you. - Please let us know of any hurdles and we will do our best to support. - -================== -Benchmark Tutorial -================== - -The Brain-Score benchmark tutorial has moved, and can be found here: https://www.brain-score.org/tutorials/benchmarks \ No newline at end of file diff --git a/docs/source/modules/developer_clarifications.rst b/docs/source/modules/developer_clarifications.rst new file mode 100644 index 000000000..6e63f2008 --- /dev/null +++ b/docs/source/modules/developer_clarifications.rst @@ -0,0 +1,36 @@ +.. _interface: + +************************ +Developer Clarifications +************************ + +The Following documentation stores commonly-asked developer questions. We hope this will be useful to +anyone interested in contributing to Brain-Score's codebase or scientific workings. + + + +1. **For a given model, are activations different on each benchmark? How?** + + + Activations per model are generated based on benchmark stimuli; not every benchmark has unique stimuli. For most + model-benchmark pairs, activations will be different because stimuli will be different. The exceptions to this + are the benchmarks that use the same stimuli, such as the `MajajHong20215` family of benchmarks. + +2. **Result Caching** + + Result Caching is a Brain-Score `repo `_ that allows model activations (and other functions) to be cached + to disk, in order to speed up the process of rescoring models. It contains a decorator that can be attached to a function + right before it is defined. On the first run of that function, `result_caching` will save to disk the result of tha function + and will load that result from disk in future calls with the same parameters. All files are saved in the user's `~/result_caching` + folder, and they are persistent, as there is no garbage collection built in. You can deactivate + `result_caching` by simply setting the environment flag `RESULTCACHING_DISABLE` to `1`. Please see the link above + for more detailed documentation. + +3. **Model Mapping Procedure** + + In general, there are different methods that are used in the Brain-Score code to instruct the model to "begin recording", + observe stimuli, and to generate scores. Models follow the `ModelCommitment` to conform to the `BrainModel` API. A + `BrainModel` is any model that has a `region_layer_map`. This allows the layers in the model to be mapped to layers in + the ventral visual stream, and is chosen by scoring models on the public version of a benchmark (the private + benchmark data is heldout for the BrainModel to be scored on). See the more technical docs + `here `_ for additional notes. diff --git a/docs/source/modules/model_tutorial.rst b/docs/source/modules/model_tutorial.rst deleted file mode 100644 index 005b3066f..000000000 --- a/docs/source/modules/model_tutorial.rst +++ /dev/null @@ -1,18 +0,0 @@ -.. _model_tutorial: - -.. _technical paper: https://www.biorxiv.org/content/10.1101/407007v1 -.. _perspective paper: https://www.cell.com/neuron/fulltext/S0896-6273(20)30605-X -.. _Pycharm: https://www.jetbrains.com/pycharm/download -.. _introduction: https://www.tutorialspoint.com/pycharm/index.htm -.. _sample-model-submission: https://github.com/brain-score/sample-model-submission -.. _github: https://github.com/brain-score -.. _github repository: https://github.com/brain-score/vision -.. _windows: https://git-scm.com/download/win -.. _mac: https://git-scm.com/download/mac -.. _profile: http://www.brain-score.org/profile/ - -============== -Model Tutorial -============== - -The Brain-Score model tutorial has moved, and can be found here: https://www.brain-score.org/tutorials/models diff --git a/pyproject.toml b/pyproject.toml index a336cc52c..3b28322e9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,7 +49,7 @@ test = [ [build-system] requires = [ - "setuptools>=65.*", + "setuptools>=65.0.0", "wheel" ]