diff --git a/.github/workflows/sync_develop_with_master.yml b/.github/workflows/sync_develop_with_master.yml index df8cc238d..bd1bf5b0c 100644 --- a/.github/workflows/sync_develop_with_master.yml +++ b/.github/workflows/sync_develop_with_master.yml @@ -16,6 +16,7 @@ jobs: id: init run: | echo "Starting branch synchronization of ${{ github.repository }}" + create_pr_for_nonplugin: name: Synchronizing non-plugin PR needs: start # This job now needs the 'start' job to complete first @@ -34,15 +35,21 @@ jobs: run: | git fetch origin master git reset --hard origin/master + - name: Get commit summary + id: commit_summary + run: | + git log -1 --pretty=format:"%s" + echo "::set-output name=summary::$(git log -1 --pretty=format:"%s")" - name: Create pull request in develop uses: peter-evans/create-pull-request@v6 with: token: '${{ secrets.PAT }}' commit-message: Sync master into develop - title: Sync master into develop + title: Sync master into develop. Triggered by PR #${{ github.event.pull_request.number }} body: >- This PR syncs the latest changes from the master branch into the develop branch. + Commit Summary: ${{ steps.commit_summary.outputs.summary }} base: develop branch: 'developer-sync-pr-${{ github.event.pull_request.number }}' @@ -85,7 +92,7 @@ jobs: fi } - name: Push changes to develop (if merge is successful) - if: steps.merge.conclusion == 'success' + if: steps.merge.outcome == 'success' run: | #Use force-with-lease to prevent accidental overwrite if branch has been updated. If fails, rebase the update and retry git push origin develop --force-with-lease || { echo "Push failed due to updates in develop. Attempting to rebase and retry..." @@ -93,14 +100,21 @@ jobs: git rebase origin/develop git push origin develop --force-with-lease } + - name: Get commit summary + id: commit_summary + run: | + git log -1 --pretty=format:"%s" + echo "::set-output name=summary::$(git log -1 --pretty=format:"%s")" - name: Create pull request for merge conflicts if: steps.merge.outputs.merge_conflict == 'true' uses: peter-evans/create-pull-request@v6 with: token: '${{ secrets.PAT }}' commit-message: Merge master into develop with conflict resolution - title: Resolve conflicts between master and develop - body: This PR resolves merge conflicts between master and develop. + title: Resolve conflicts between master and develop. Triggered by PR #${{ github.event.pull_request.number }} + body: | + This PR resolves merge conflicts between master and develop. + Commit Summary: ${{ steps.commit_summary.outputs.summary }} base: develop branch: 'developer-sync-pr-conflict-${{ github.event.pull_request.number }}' - name: Handle other merge failures @@ -108,4 +122,18 @@ jobs: run: > echo "Handle non-conflict related failure, such as network issues or missing branches" - # Possibly incorporate additional handling logic here (e.g.,notifications or retries) \ No newline at end of file + # Possibly incorporate additional handling logic here (e.g.,notifications or retries) + + + no_changes: + name: "No Changes Made. No synchronization needed." + needs: start + if: > + ( + needs.create_pr_for_nonplugin.result != 'success' && + needs.auto_sync_for_plugin.result != 'success' + ) + runs-on: ubuntu-latest + steps: + - name: Echo no changes + run: echo "No changes were made to master branch 👍" diff --git a/brainscore_vision/models/scaling_models/__init__.py b/brainscore_vision/models/scaling_models/__init__.py new file mode 100644 index 000000000..e020a07e5 --- /dev/null +++ b/brainscore_vision/models/scaling_models/__init__.py @@ -0,0 +1,265 @@ +from brainscore_vision import model_registry +from brainscore_vision.model_helpers.brain_transformation import ModelCommitment +from .model import get_model, MODEL_CONFIGS + +model_registry["resnet18_imagenet_full"] = lambda: ModelCommitment( + identifier="resnet18_imagenet_full", + activations_model=get_model("resnet18_imagenet_full"), + layers=MODEL_CONFIGS["resnet18_imagenet_full"]["model_commitment"]["layers"], + behavioral_readout_layer=MODEL_CONFIGS["resnet18_imagenet_full"]["model_commitment"]["behavioral_readout_layer"], + region_layer_map=MODEL_CONFIGS["resnet18_imagenet_full"]["model_commitment"]["region_layer_map"] +) + + +model_registry["resnet34_imagenet_full"] = lambda: ModelCommitment( + identifier="resnet34_imagenet_full", + activations_model=get_model("resnet34_imagenet_full"), + layers=MODEL_CONFIGS["resnet34_imagenet_full"]["model_commitment"]["layers"], + behavioral_readout_layer=MODEL_CONFIGS["resnet34_imagenet_full"]["model_commitment"]["behavioral_readout_layer"], + region_layer_map=MODEL_CONFIGS["resnet34_imagenet_full"]["model_commitment"]["region_layer_map"] +) + + +model_registry["resnet50_imagenet_full"] = lambda: ModelCommitment( + identifier="resnet50_imagenet_full", + activations_model=get_model("resnet50_imagenet_full"), + layers=MODEL_CONFIGS["resnet50_imagenet_full"]["model_commitment"]["layers"], + behavioral_readout_layer=MODEL_CONFIGS["resnet50_imagenet_full"]["model_commitment"]["behavioral_readout_layer"], + region_layer_map=MODEL_CONFIGS["resnet50_imagenet_full"]["model_commitment"]["region_layer_map"] +) + + +model_registry["resnet101_imagenet_full"] = lambda: ModelCommitment( + identifier="resnet101_imagenet_full", + activations_model=get_model("resnet101_imagenet_full"), + layers=MODEL_CONFIGS["resnet101_imagenet_full"]["model_commitment"]["layers"], + behavioral_readout_layer=MODEL_CONFIGS["resnet101_imagenet_full"]["model_commitment"]["behavioral_readout_layer"], + region_layer_map=MODEL_CONFIGS["resnet101_imagenet_full"]["model_commitment"]["region_layer_map"] +) + + +model_registry["resnet152_imagenet_full"] = lambda: ModelCommitment( + identifier="resnet152_imagenet_full", + activations_model=get_model("resnet152_imagenet_full"), + layers=MODEL_CONFIGS["resnet152_imagenet_full"]["model_commitment"]["layers"], + behavioral_readout_layer=MODEL_CONFIGS["resnet152_imagenet_full"]["model_commitment"]["behavioral_readout_layer"], + region_layer_map=MODEL_CONFIGS["resnet152_imagenet_full"]["model_commitment"]["region_layer_map"] +) + + +model_registry["resnet18_ecoset_full"] = lambda: ModelCommitment( + identifier="resnet18_ecoset_full", + activations_model=get_model("resnet18_ecoset_full"), + layers=MODEL_CONFIGS["resnet18_ecoset_full"]["model_commitment"]["layers"], + behavioral_readout_layer=MODEL_CONFIGS["resnet18_ecoset_full"]["model_commitment"]["behavioral_readout_layer"], + region_layer_map=MODEL_CONFIGS["resnet18_ecoset_full"]["model_commitment"]["region_layer_map"] +) + + +model_registry["resnet34_ecoset_full"] = lambda: ModelCommitment( + identifier="resnet34_ecoset_full", + activations_model=get_model("resnet34_ecoset_full"), + layers=MODEL_CONFIGS["resnet34_ecoset_full"]["model_commitment"]["layers"], + behavioral_readout_layer=MODEL_CONFIGS["resnet34_ecoset_full"]["model_commitment"]["behavioral_readout_layer"], + region_layer_map=MODEL_CONFIGS["resnet34_ecoset_full"]["model_commitment"]["region_layer_map"] +) + + +model_registry["resnet50_ecoset_full"] = lambda: ModelCommitment( + identifier="resnet50_ecoset_full", + activations_model=get_model("resnet50_ecoset_full"), + layers=MODEL_CONFIGS["resnet50_ecoset_full"]["model_commitment"]["layers"], + behavioral_readout_layer=MODEL_CONFIGS["resnet50_ecoset_full"]["model_commitment"]["behavioral_readout_layer"], + region_layer_map=MODEL_CONFIGS["resnet50_ecoset_full"]["model_commitment"]["region_layer_map"] +) + + +model_registry["resnet101_ecoset_full"] = lambda: ModelCommitment( + identifier="resnet101_ecoset_full", + activations_model=get_model("resnet101_ecoset_full"), + layers=MODEL_CONFIGS["resnet101_ecoset_full"]["model_commitment"]["layers"], + behavioral_readout_layer=MODEL_CONFIGS["resnet101_ecoset_full"]["model_commitment"]["behavioral_readout_layer"], + region_layer_map=MODEL_CONFIGS["resnet101_ecoset_full"]["model_commitment"]["region_layer_map"] +) + + +model_registry["resnet152_ecoset_full"] = lambda: ModelCommitment( + identifier="resnet152_ecoset_full", + activations_model=get_model("resnet152_ecoset_full"), + layers=MODEL_CONFIGS["resnet152_ecoset_full"]["model_commitment"]["layers"], + behavioral_readout_layer=MODEL_CONFIGS["resnet152_ecoset_full"]["model_commitment"]["behavioral_readout_layer"], + region_layer_map=MODEL_CONFIGS["resnet152_ecoset_full"]["model_commitment"]["region_layer_map"] +) + + +model_registry["resnet50_imagenet_1_seed-0"] = lambda: ModelCommitment( + identifier="resnet50_imagenet_1_seed-0", + activations_model=get_model("resnet50_imagenet_1_seed-0"), + layers=MODEL_CONFIGS["resnet50_imagenet_1_seed-0"]["model_commitment"]["layers"], + behavioral_readout_layer=MODEL_CONFIGS["resnet50_imagenet_1_seed-0"]["model_commitment"]["behavioral_readout_layer"], + region_layer_map=MODEL_CONFIGS["resnet50_imagenet_1_seed-0"]["model_commitment"]["region_layer_map"] +) + + +model_registry["resnet50_imagenet_10_seed-0"] = lambda: ModelCommitment( + identifier="resnet50_imagenet_10_seed-0", + activations_model=get_model("resnet50_imagenet_10_seed-0"), + layers=MODEL_CONFIGS["resnet50_imagenet_10_seed-0"]["model_commitment"]["layers"], + behavioral_readout_layer=MODEL_CONFIGS["resnet50_imagenet_10_seed-0"]["model_commitment"]["behavioral_readout_layer"], + region_layer_map=MODEL_CONFIGS["resnet50_imagenet_10_seed-0"]["model_commitment"]["region_layer_map"] +) + + +model_registry["resnet50_imagenet_100_seed-0"] = lambda: ModelCommitment( + identifier="resnet50_imagenet_100_seed-0", + activations_model=get_model("resnet50_imagenet_100_seed-0"), + layers=MODEL_CONFIGS["resnet50_imagenet_100_seed-0"]["model_commitment"]["layers"], + behavioral_readout_layer=MODEL_CONFIGS["resnet50_imagenet_100_seed-0"]["model_commitment"]["behavioral_readout_layer"], + region_layer_map=MODEL_CONFIGS["resnet50_imagenet_100_seed-0"]["model_commitment"]["region_layer_map"] +) + + +model_registry["efficientnet_b0_imagenet_full"] = lambda: ModelCommitment( + identifier="efficientnet_b0_imagenet_full", + activations_model=get_model("efficientnet_b0_imagenet_full"), + layers=MODEL_CONFIGS["efficientnet_b0_imagenet_full"]["model_commitment"]["layers"], + behavioral_readout_layer=MODEL_CONFIGS["efficientnet_b0_imagenet_full"]["model_commitment"]["behavioral_readout_layer"], + region_layer_map=MODEL_CONFIGS["efficientnet_b0_imagenet_full"]["model_commitment"]["region_layer_map"] +) + + +model_registry["efficientnet_b1_imagenet_full"] = lambda: ModelCommitment( + identifier="efficientnet_b1_imagenet_full", + activations_model=get_model("efficientnet_b1_imagenet_full"), + layers=MODEL_CONFIGS["efficientnet_b1_imagenet_full"]["model_commitment"]["layers"], + behavioral_readout_layer=MODEL_CONFIGS["efficientnet_b1_imagenet_full"]["model_commitment"]["behavioral_readout_layer"], + region_layer_map=MODEL_CONFIGS["efficientnet_b1_imagenet_full"]["model_commitment"]["region_layer_map"] +) + + +model_registry["efficientnet_b2_imagenet_full"] = lambda: ModelCommitment( + identifier="efficientnet_b2_imagenet_full", + activations_model=get_model("efficientnet_b2_imagenet_full"), + layers=MODEL_CONFIGS["efficientnet_b2_imagenet_full"]["model_commitment"]["layers"], + behavioral_readout_layer=MODEL_CONFIGS["efficientnet_b2_imagenet_full"]["model_commitment"]["behavioral_readout_layer"], + region_layer_map=MODEL_CONFIGS["efficientnet_b2_imagenet_full"]["model_commitment"]["region_layer_map"] +) + + +model_registry["deit_small_imagenet_full_seed-0"] = lambda: ModelCommitment( + identifier="deit_small_imagenet_full_seed-0", + activations_model=get_model("deit_small_imagenet_full_seed-0"), + layers=MODEL_CONFIGS["deit_small_imagenet_full_seed-0"]["model_commitment"]["layers"], + behavioral_readout_layer=MODEL_CONFIGS["deit_small_imagenet_full_seed-0"]["model_commitment"]["behavioral_readout_layer"], + region_layer_map=MODEL_CONFIGS["deit_small_imagenet_full_seed-0"]["model_commitment"]["region_layer_map"] +) + + +model_registry["deit_base_imagenet_full_seed-0"] = lambda: ModelCommitment( + identifier="deit_base_imagenet_full_seed-0", + activations_model=get_model("deit_base_imagenet_full_seed-0"), + layers=MODEL_CONFIGS["deit_base_imagenet_full_seed-0"]["model_commitment"]["layers"], + behavioral_readout_layer=MODEL_CONFIGS["deit_base_imagenet_full_seed-0"]["model_commitment"]["behavioral_readout_layer"], + region_layer_map=MODEL_CONFIGS["deit_base_imagenet_full_seed-0"]["model_commitment"]["region_layer_map"] +) + + +model_registry["deit_large_imagenet_full_seed-0"] = lambda: ModelCommitment( + identifier="deit_large_imagenet_full_seed-0", + activations_model=get_model("deit_large_imagenet_full_seed-0"), + layers=MODEL_CONFIGS["deit_large_imagenet_full_seed-0"]["model_commitment"]["layers"], + behavioral_readout_layer=MODEL_CONFIGS["deit_large_imagenet_full_seed-0"]["model_commitment"]["behavioral_readout_layer"], + region_layer_map=MODEL_CONFIGS["deit_large_imagenet_full_seed-0"]["model_commitment"]["region_layer_map"] +) + + +model_registry["deit_small_imagenet_1_seed-0"] = lambda: ModelCommitment( + identifier="deit_small_imagenet_1_seed-0", + activations_model=get_model("deit_small_imagenet_1_seed-0"), + layers=MODEL_CONFIGS["deit_small_imagenet_1_seed-0"]["model_commitment"]["layers"], + behavioral_readout_layer=MODEL_CONFIGS["deit_small_imagenet_1_seed-0"]["model_commitment"]["behavioral_readout_layer"], + region_layer_map=MODEL_CONFIGS["deit_small_imagenet_1_seed-0"]["model_commitment"]["region_layer_map"] +) + + +model_registry["deit_small_imagenet_10_seed-0"] = lambda: ModelCommitment( + identifier="deit_small_imagenet_10_seed-0", + activations_model=get_model("deit_small_imagenet_10_seed-0"), + layers=MODEL_CONFIGS["deit_small_imagenet_10_seed-0"]["model_commitment"]["layers"], + behavioral_readout_layer=MODEL_CONFIGS["deit_small_imagenet_10_seed-0"]["model_commitment"]["behavioral_readout_layer"], + region_layer_map=MODEL_CONFIGS["deit_small_imagenet_10_seed-0"]["model_commitment"]["region_layer_map"] +) + + +model_registry["deit_small_imagenet_100_seed-0"] = lambda: ModelCommitment( + identifier="deit_small_imagenet_100_seed-0", + activations_model=get_model("deit_small_imagenet_100_seed-0"), + layers=MODEL_CONFIGS["deit_small_imagenet_100_seed-0"]["model_commitment"]["layers"], + behavioral_readout_layer=MODEL_CONFIGS["deit_small_imagenet_100_seed-0"]["model_commitment"]["behavioral_readout_layer"], + region_layer_map=MODEL_CONFIGS["deit_small_imagenet_100_seed-0"]["model_commitment"]["region_layer_map"] +) + + +model_registry["convnext_tiny_imagenet_full_seed-0"] = lambda: ModelCommitment( + identifier="convnext_tiny_imagenet_full_seed-0", + activations_model=get_model("convnext_tiny_imagenet_full_seed-0"), + layers=MODEL_CONFIGS["convnext_tiny_imagenet_full_seed-0"]["model_commitment"]["layers"], + behavioral_readout_layer=MODEL_CONFIGS["convnext_tiny_imagenet_full_seed-0"]["model_commitment"]["behavioral_readout_layer"], + region_layer_map=MODEL_CONFIGS["convnext_tiny_imagenet_full_seed-0"]["model_commitment"]["region_layer_map"] +) + + +model_registry["convnext_small_imagenet_full_seed-0"] = lambda: ModelCommitment( + identifier="convnext_small_imagenet_full_seed-0", + activations_model=get_model("convnext_small_imagenet_full_seed-0"), + layers=MODEL_CONFIGS["convnext_small_imagenet_full_seed-0"]["model_commitment"]["layers"], + behavioral_readout_layer=MODEL_CONFIGS["convnext_small_imagenet_full_seed-0"]["model_commitment"]["behavioral_readout_layer"], + region_layer_map=MODEL_CONFIGS["convnext_small_imagenet_full_seed-0"]["model_commitment"]["region_layer_map"] +) + + +model_registry["convnext_base_imagenet_full_seed-0"] = lambda: ModelCommitment( + identifier="convnext_base_imagenet_full_seed-0", + activations_model=get_model("convnext_base_imagenet_full_seed-0"), + layers=MODEL_CONFIGS["convnext_base_imagenet_full_seed-0"]["model_commitment"]["layers"], + behavioral_readout_layer=MODEL_CONFIGS["convnext_base_imagenet_full_seed-0"]["model_commitment"]["behavioral_readout_layer"], + region_layer_map=MODEL_CONFIGS["convnext_base_imagenet_full_seed-0"]["model_commitment"]["region_layer_map"] +) + + +model_registry["convnext_large_imagenet_full_seed-0"] = lambda: ModelCommitment( + identifier="convnext_large_imagenet_full_seed-0", + activations_model=get_model("convnext_large_imagenet_full_seed-0"), + layers=MODEL_CONFIGS["convnext_large_imagenet_full_seed-0"]["model_commitment"]["layers"], + behavioral_readout_layer=MODEL_CONFIGS["convnext_large_imagenet_full_seed-0"]["model_commitment"]["behavioral_readout_layer"], + region_layer_map=MODEL_CONFIGS["convnext_large_imagenet_full_seed-0"]["model_commitment"]["region_layer_map"] +) + + +model_registry["convnext_small_imagenet_1_seed-0"] = lambda: ModelCommitment( + identifier="convnext_small_imagenet_1_seed-0", + activations_model=get_model("convnext_small_imagenet_1_seed-0"), + layers=MODEL_CONFIGS["convnext_small_imagenet_1_seed-0"]["model_commitment"]["layers"], + behavioral_readout_layer=MODEL_CONFIGS["convnext_small_imagenet_1_seed-0"]["model_commitment"]["behavioral_readout_layer"], + region_layer_map=MODEL_CONFIGS["convnext_small_imagenet_1_seed-0"]["model_commitment"]["region_layer_map"] +) + + +model_registry["convnext_small_imagenet_10_seed-0"] = lambda: ModelCommitment( + identifier="convnext_small_imagenet_10_seed-0", + activations_model=get_model("convnext_small_imagenet_10_seed-0"), + layers=MODEL_CONFIGS["convnext_small_imagenet_10_seed-0"]["model_commitment"]["layers"], + behavioral_readout_layer=MODEL_CONFIGS["convnext_small_imagenet_10_seed-0"]["model_commitment"]["behavioral_readout_layer"], + region_layer_map=MODEL_CONFIGS["convnext_small_imagenet_10_seed-0"]["model_commitment"]["region_layer_map"] +) + + +model_registry["convnext_small_imagenet_100_seed-0"] = lambda: ModelCommitment( + identifier="convnext_small_imagenet_100_seed-0", + activations_model=get_model("convnext_small_imagenet_100_seed-0"), + layers=MODEL_CONFIGS["convnext_small_imagenet_100_seed-0"]["model_commitment"]["layers"], + behavioral_readout_layer=MODEL_CONFIGS["convnext_small_imagenet_100_seed-0"]["model_commitment"]["behavioral_readout_layer"], + region_layer_map=MODEL_CONFIGS["convnext_small_imagenet_100_seed-0"]["model_commitment"]["region_layer_map"] +) + + diff --git a/brainscore_vision/models/scaling_models/model.py b/brainscore_vision/models/scaling_models/model.py new file mode 100644 index 000000000..c7a470e6e --- /dev/null +++ b/brainscore_vision/models/scaling_models/model.py @@ -0,0 +1,148 @@ +import os +import functools +import json +from pathlib import Path +import ssl + +import torchvision.models +import torch + +from brainscore_vision.model_helpers.activations.pytorch import PytorchWrapper +from brainscore_vision.model_helpers.activations.pytorch import load_preprocess_images + +import timm +import numpy as np +import torchvision.transforms as T +from PIL import Image + +import albumentations as A +from albumentations.pytorch import ToTensorV2 + +# Disable SSL verification +ssl._create_default_https_context = ssl._create_unverified_context + +BIBTEX = """""" + + +with open(Path(__file__).parent / "model_configs.json", "r") as f: + MODEL_CONFIGS = json.load(f) + + +def load_image(image_filepath): + return Image.open(image_filepath).convert("RGB") + + +def get_interpolation_mode(interpolation: str) -> int: + """Returns the interpolation mode for albumentations""" + if "linear" or "bilinear" in interpolation: + return 1 + elif "cubic" or "bicubic" in interpolation: + return 2 + else: + raise NotImplementedError(f"Interpolation mode {interpolation} not implemented") + + +def custom_image_preprocess( + images, + resize_size: int, + crop_size: int, + interpolation: str, + transforms=None, +): + if transforms is None: + interpolation = get_interpolation_mode(interpolation) + transforms = A.Compose( + [ + A.Resize(resize_size, resize_size, p=1.0, interpolation=interpolation), + A.CenterCrop(crop_size, crop_size, p=1.0), + A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), + ToTensorV2(), + ] + ) + if isinstance(transforms, T.Compose): + images = [transforms(image) for image in images] + images = [np.array(image) for image in images] + images = np.stack(images) + elif isinstance(transforms, A.Compose): + images = [transforms(image=np.array(image))["image"] for image in images] + images = np.stack(images) + else: + raise NotImplementedError( + f"Transform of type {type(transforms)} is not implemented" + ) + + return images + + +def load_preprocess_images_custom( + image_filepaths, preprocess_images=custom_image_preprocess, **kwargs +): + images = [load_image(image_filepath) for image_filepath in image_filepaths] + images = preprocess_images(images, **kwargs) + return images + + +def get_model(model_id:str): + + # Unpack model config + config = MODEL_CONFIGS[model_id] + model_name = config["model_name"] + model_id = config["model_id"] + resize_size = config["resize_size"] + crop_size = config["crop_size"] + interpolation = config["interpolation"] + num_classes = config["num_classes"] + ckpt_url = config["checkpoint_url"] + use_timm = config["use_timm"] + timm_model_name = config["timm_model_name"] + epoch = config["epoch"] + load_model_ema = config["load_model_ema"] + output_head = config["output_head"] + is_vit = config["is_vit"] + + # Temporary fix for vit models + # See https://github.com/brain-score/vision/pull/1232 + if is_vit: + os.environ['RESULTCACHING_DISABLE'] = 'brainscore_vision.model_helpers.activations.core.ActivationsExtractorHelper._from_paths_stored' + + + # Initialize model + if use_timm: + model = timm.create_model(timm_model_name, pretrained=False, num_classes=num_classes) + else: + model = eval(f"torchvision.models.{model_name}(weights=None)") + if num_classes != 1000: + exec(f'''{output_head} = torch.nn.Linear( + in_features={output_head}.in_features, + out_features=num_classes, + bias={output_head}.bias is not None, + )''' + ) + + # Load model weights + state_dict = torch.hub.load_state_dict_from_url( + ckpt_url, + check_hash=True, + file_name=f"{model_id}_ep{epoch}.pt", + map_location="cpu", + ) + if load_model_ema: + state_dict = state_dict["state"]["model_ema_state_dict"] + else: + state_dict = state_dict["state"]["model"] + state_dict = {k.replace("module.", ""): v for k, v in state_dict.items()} + model.load_state_dict(state_dict, strict=True) + print(f"Model loaded from {ckpt_url}") + + # Wrap model + preprocessing = functools.partial( + load_preprocess_images_custom, + resize_size=resize_size, + crop_size=crop_size, + interpolation=interpolation, + transforms=None + ) + wrapper = PytorchWrapper( + identifier=model_id, model=model, preprocessing=preprocessing + ) + return wrapper diff --git a/brainscore_vision/models/scaling_models/model_configs.json b/brainscore_vision/models/scaling_models/model_configs.json new file mode 100644 index 000000000..cc52e5370 --- /dev/null +++ b/brainscore_vision/models/scaling_models/model_configs.json @@ -0,0 +1,869 @@ +{ + "resnet18_imagenet_full": { + "model_name": "resnet18", + "model_id": "resnet18_imagenet_full", + "num_classes": 1000, + "resize_size": 256, + "crop_size": 224, + "interpolation": "bilinear", + "load_model_ema": false, + "use_timm": false, + "timm_model_name": null, + "is_vit": false, + "epoch": 100, + "output_head": "model.fc", + "model_commitment": { + "layers": [ + "layer1.0.bn1", + "layer2.0.bn2", + "layer3.0.conv2", + "layer4.0.bn1" + ], + "behavioral_readout_layer": "avgpool", + "region_layer_map": { + "V1": "layer1.0.bn1", + "V2": "layer3.0.conv2", + "V4": "layer2.0.bn2", + "IT": "layer4.0.bn1" + } + }, + "checkpoint_url": "https://epfl-neuroailab-scalinglaws.s3.eu-north-1.amazonaws.com/checkpoints/resnet18_imagenet_full/ep100.pt" + }, + "resnet34_imagenet_full": { + "model_name": "resnet34", + "model_id": "resnet34_imagenet_full", + "num_classes": 1000, + "resize_size": 256, + "crop_size": 224, + "interpolation": "bilinear", + "load_model_ema": false, + "use_timm": false, + "timm_model_name": null, + "is_vit": false, + "epoch": 100, + "output_head": "model.fc", + "model_commitment": { + "layers": [ + "layer1.0.bn1", + "layer2.0.bn2", + "layer3.1.conv1", + "layer4.0.bn1" + ], + "behavioral_readout_layer": "avgpool", + "region_layer_map": { + "V1": "layer1.0.bn1", + "V2": "layer3.1.conv1", + "V4": "layer2.0.bn2", + "IT": "layer4.0.bn1" + } + }, + "checkpoint_url": "https://epfl-neuroailab-scalinglaws.s3.eu-north-1.amazonaws.com/checkpoints/resnet34_imagenet_full/ep100.pt" + }, + "resnet50_imagenet_full": { + "model_name": "resnet50", + "model_id": "resnet50_imagenet_full", + "num_classes": 1000, + "resize_size": 256, + "crop_size": 224, + "interpolation": "bilinear", + "load_model_ema": false, + "use_timm": false, + "timm_model_name": null, + "is_vit": false, + "epoch": 100, + "output_head": "model.fc", + "model_commitment": { + "layers": [ + "layer1.0.conv1", + "layer3.0.conv1", + "layer3.5.bn3", + "layer4.0.relu" + ], + "behavioral_readout_layer": "avgpool", + "region_layer_map": { + "V1": "layer1.0.conv1", + "V2": "layer3.5.bn3", + "V4": "layer3.0.conv1", + "IT": "layer4.0.relu" + } + }, + "checkpoint_url": "https://epfl-neuroailab-scalinglaws.s3.eu-north-1.amazonaws.com/checkpoints/resnet50_imagenet_full/ep100.pt" + }, + "resnet101_imagenet_full": { + "model_name": "resnet101", + "model_id": "resnet101_imagenet_full", + "num_classes": 1000, + "resize_size": 256, + "crop_size": 224, + "interpolation": "bilinear", + "load_model_ema": false, + "use_timm": false, + "timm_model_name": null, + "is_vit": false, + "epoch": 100, + "output_head": "model.fc", + "model_commitment": { + "layers": [ + "layer1.0.bn1", + "layer3.0.bn3", + "layer4.0.bn1", + "layer4.0.relu" + ], + "behavioral_readout_layer": "avgpool", + "region_layer_map": { + "V1": "layer1.0.bn1", + "V2": "layer4.0.bn1", + "V4": "layer3.0.bn3", + "IT": "layer4.0.relu" + } + }, + "checkpoint_url": "https://epfl-neuroailab-scalinglaws.s3.eu-north-1.amazonaws.com/checkpoints/resnet101_imagenet_full/ep100.pt" + }, + "resnet152_imagenet_full": { + "model_name": "resnet152", + "model_id": "resnet152_imagenet_full", + "num_classes": 1000, + "resize_size": 256, + "crop_size": 224, + "interpolation": "bilinear", + "load_model_ema": false, + "use_timm": false, + "timm_model_name": null, + "is_vit": false, + "epoch": 100, + "output_head": "model.fc", + "model_commitment": { + "layers": [ + "layer1.0.bn1", + "layer3.0.bn3", + "layer3.3.bn3", + "layer3.34.bn3" + ], + "behavioral_readout_layer": "avgpool", + "region_layer_map": { + "V1": "layer1.0.bn1", + "V2": "layer3.3.bn3", + "V4": "layer3.0.bn3", + "IT": "layer3.34.bn3" + } + }, + "checkpoint_url": "https://epfl-neuroailab-scalinglaws.s3.eu-north-1.amazonaws.com/checkpoints/resnet152_imagenet_full/ep100.pt" + }, + "resnet18_ecoset_full": { + "model_name": "resnet18", + "model_id": "resnet18_ecoset_full", + "num_classes": 565, + "resize_size": 256, + "crop_size": 224, + "interpolation": "bilinear", + "load_model_ema": false, + "use_timm": false, + "timm_model_name": null, + "is_vit": false, + "epoch": 100, + "output_head": "model.fc", + "model_commitment": { + "layers": [ + "layer1.0.conv1", + "layer2.0.bn2", + "layer3.0.conv1", + "layer4.0.bn1" + ], + "behavioral_readout_layer": "avgpool", + "region_layer_map": { + "V1": "layer1.0.conv1", + "V2": "layer3.0.conv1", + "V4": "layer2.0.bn2", + "IT": "layer4.0.bn1" + } + }, + "checkpoint_url": "https://epfl-neuroailab-scalinglaws.s3.eu-north-1.amazonaws.com/checkpoints/resnet18_ecoset_full/ep100.pt" + }, + "resnet34_ecoset_full": { + "model_name": "resnet34", + "model_id": "resnet34_ecoset_full", + "num_classes": 565, + "resize_size": 256, + "crop_size": 224, + "interpolation": "bilinear", + "load_model_ema": false, + "use_timm": false, + "timm_model_name": null, + "is_vit": false, + "epoch": 100, + "output_head": "model.fc", + "model_commitment": { + "layers": [ + "layer1.0.bn1", + "layer3.0.conv1", + "layer3.1.conv1", + "layer4.0.conv1" + ], + "behavioral_readout_layer": "avgpool", + "region_layer_map": { + "V1": "layer1.0.bn1", + "V2": "layer3.1.conv1", + "V4": "layer3.0.conv1", + "IT": "layer4.0.conv1" + } + }, + "checkpoint_url": "https://epfl-neuroailab-scalinglaws.s3.eu-north-1.amazonaws.com/checkpoints/resnet34_ecoset_full/ep100.pt" + }, + "resnet50_ecoset_full": { + "model_name": "resnet50", + "model_id": "resnet50_ecoset_full", + "num_classes": 565, + "resize_size": 256, + "crop_size": 224, + "interpolation": "bilinear", + "load_model_ema": false, + "use_timm": false, + "timm_model_name": null, + "is_vit": false, + "epoch": 100, + "output_head": "model.fc", + "model_commitment": { + "layers": [ + "layer1.0.bn1", + "layer3.0.conv1", + "layer4.0.conv2", + "layer4.0.relu" + ], + "behavioral_readout_layer": "avgpool", + "region_layer_map": { + "V1": "layer1.0.bn1", + "V2": "layer4.0.conv2", + "V4": "layer3.0.conv1", + "IT": "layer4.0.relu" + } + }, + "checkpoint_url": "https://epfl-neuroailab-scalinglaws.s3.eu-north-1.amazonaws.com/checkpoints/resnet50_ecoset_full/ep100.pt" + }, + "resnet101_ecoset_full": { + "model_name": "resnet101", + "model_id": "resnet101_ecoset_full", + "num_classes": 565, + "resize_size": 256, + "crop_size": 224, + "interpolation": "bilinear", + "load_model_ema": false, + "use_timm": false, + "timm_model_name": null, + "is_vit": false, + "epoch": 100, + "output_head": "model.fc", + "model_commitment": { + "layers": [ + "layer1.0.bn1", + "layer3.0.bn3", + "layer3.4.relu", + "layer4.0.relu" + ], + "behavioral_readout_layer": "avgpool", + "region_layer_map": { + "V1": "layer1.0.bn1", + "V2": "layer3.4.relu", + "V4": "layer3.0.bn3", + "IT": "layer4.0.relu" + } + }, + "checkpoint_url": "https://epfl-neuroailab-scalinglaws.s3.eu-north-1.amazonaws.com/checkpoints/resnet101_ecoset_full/ep100.pt" + }, + "resnet152_ecoset_full": { + "model_name": "resnet152", + "model_id": "resnet152_ecoset_full", + "num_classes": 565, + "resize_size": 256, + "crop_size": 224, + "interpolation": "bilinear", + "load_model_ema": false, + "use_timm": false, + "timm_model_name": null, + "is_vit": false, + "epoch": 100, + "output_head": "model.fc", + "model_commitment": { + "layers": [ + "layer1.0.bn1", + "layer3.0.bn3", + "layer3.3.bn3", + "layer4.0.relu" + ], + "behavioral_readout_layer": "avgpool", + "region_layer_map": { + "V1": "layer1.0.bn1", + "V2": "layer3.3.bn3", + "V4": "layer3.0.bn3", + "IT": "layer4.0.relu" + } + }, + "checkpoint_url": "https://epfl-neuroailab-scalinglaws.s3.eu-north-1.amazonaws.com/checkpoints/resnet152_ecoset_full/ep100.pt" + }, + "resnet50_imagenet_1_seed-0": { + "model_name": "resnet50", + "model_id": "resnet50_imagenet_1_seed-0", + "num_classes": 1000, + "resize_size": 256, + "crop_size": 224, + "interpolation": "bilinear", + "load_model_ema": false, + "use_timm": false, + "timm_model_name": null, + "is_vit": false, + "epoch": 100, + "output_head": "model.fc", + "model_commitment": { + "layers": [ + "layer1.0.conv1", + "layer3.0.conv1", + "layer3.5.bn3", + "layer4.0.relu" + ], + "behavioral_readout_layer": "avgpool", + "region_layer_map": { + "V1": "layer1.0.conv1", + "V2": "layer3.5.bn3", + "V4": "layer3.0.conv1", + "IT": "layer4.0.relu" + } + }, + "checkpoint_url": "https://epfl-neuroailab-scalinglaws.s3.eu-north-1.amazonaws.com/checkpoints/resnet50_imagenet_1_seed-0/ep100.pt" + }, + "resnet50_imagenet_10_seed-0": { + "model_name": "resnet50", + "model_id": "resnet50_imagenet_10_seed-0", + "num_classes": 1000, + "resize_size": 256, + "crop_size": 224, + "interpolation": "bilinear", + "load_model_ema": false, + "use_timm": false, + "timm_model_name": null, + "is_vit": false, + "epoch": 100, + "output_head": "model.fc", + "model_commitment": { + "layers": [ + "layer1.0.conv1", + "layer3.0.conv1", + "layer3.5.bn3", + "layer4.0.relu" + ], + "behavioral_readout_layer": "avgpool", + "region_layer_map": { + "V1": "layer1.0.conv1", + "V2": "layer3.5.bn3", + "V4": "layer3.0.conv1", + "IT": "layer4.0.relu" + } + }, + "checkpoint_url": "https://epfl-neuroailab-scalinglaws.s3.eu-north-1.amazonaws.com/checkpoints/resnet50_imagenet_10_seed-0/ep100.pt" + }, + "resnet50_imagenet_100_seed-0": { + "model_name": "resnet50", + "model_id": "resnet50_imagenet_100_seed-0", + "num_classes": 1000, + "resize_size": 256, + "crop_size": 224, + "interpolation": "bilinear", + "load_model_ema": false, + "use_timm": false, + "timm_model_name": null, + "is_vit": false, + "epoch": 100, + "output_head": "model.fc", + "model_commitment": { + "layers": [ + "layer1.0.conv1", + "layer3.0.conv1", + "layer3.5.bn3", + "layer4.0.relu" + ], + "behavioral_readout_layer": "avgpool", + "region_layer_map": { + "V1": "layer1.0.conv1", + "V2": "layer3.5.bn3", + "V4": "layer3.0.conv1", + "IT": "layer4.0.relu" + } + }, + "checkpoint_url": "https://epfl-neuroailab-scalinglaws.s3.eu-north-1.amazonaws.com/checkpoints/resnet50_imagenet_100_seed-0/ep100.pt" + }, + "efficientnet_b0_imagenet_full": { + "model_name": "efficientnet_b0", + "model_id": "efficientnet_b0_imagenet_full", + "num_classes": 1000, + "resize_size": 256, + "crop_size": 224, + "interpolation": "bilinear", + "load_model_ema": false, + "use_timm": false, + "timm_model_name": null, + "is_vit": false, + "epoch": 100, + "output_head": "model.classifier[1]", + "model_commitment": { + "layers": [ + "features.4.0.block.0.1", + "features.4.0.block.1.0", + "features.4.1.block.3.1", + "features.6.0.block.3.0" + ], + "behavioral_readout_layer": "avgpool", + "region_layer_map": { + "V1": "features.4.0.block.1.0", + "V2": "features.4.1.block.3.1", + "V4": "features.4.0.block.0.1", + "IT": "features.6.0.block.3.0" + } + }, + "checkpoint_url": "https://epfl-neuroailab-scalinglaws.s3.eu-north-1.amazonaws.com/checkpoints/efficientnet_b0_imagenet_full/ep100.pt" + }, + "efficientnet_b1_imagenet_full": { + "model_name": "efficientnet_b1", + "model_id": "efficientnet_b1_imagenet_full", + "num_classes": 1000, + "resize_size": 255, + "crop_size": 240, + "interpolation": "bilinear", + "load_model_ema": false, + "use_timm": false, + "timm_model_name": null, + "is_vit": false, + "epoch": 100, + "output_head": "model.classifier[1]", + "model_commitment": { + "layers": [ + "features.4.0.block.0.1", + "features.4.0.block.1.0", + "features.6.0.block.3.0" + ], + "behavioral_readout_layer": "avgpool", + "region_layer_map": { + "V1": "features.4.0.block.0.1", + "V2": "features.4.0.block.1.0", + "V4": "features.4.0.block.0.1", + "IT": "features.6.0.block.3.0" + } + }, + "checkpoint_url": "https://epfl-neuroailab-scalinglaws.s3.eu-north-1.amazonaws.com/checkpoints/efficientnet_b1_imagenet_full/ep100.pt" + }, + "efficientnet_b2_imagenet_full": { + "model_name": "efficientnet_b2", + "model_id": "efficientnet_b2_imagenet_full", + "num_classes": 1000, + "resize_size": 288, + "crop_size": 288, + "interpolation": "bilinear", + "load_model_ema": false, + "use_timm": false, + "timm_model_name": null, + "is_vit": false, + "epoch": 100, + "output_head": "model.classifier[1]", + "model_commitment": { + "layers": [ + "features.4.0.block.3.0", + "features.5.0.block.1.0", + "features.6.0.block.3.0" + ], + "behavioral_readout_layer": "avgpool", + "region_layer_map": { + "V1": "features.4.0.block.3.0", + "V2": "features.5.0.block.1.0", + "V4": "features.4.0.block.3.0", + "IT": "features.6.0.block.3.0" + } + }, + "checkpoint_url": "https://epfl-neuroailab-scalinglaws.s3.eu-north-1.amazonaws.com/checkpoints/efficientnet_b2_imagenet_full/ep100.pt" + }, + "deit_small_imagenet_full_seed-0": { + "model_name": "deit_small", + "model_id": "deit_small_imagenet_full_seed-0", + "num_classes": 1000, + "resize_size": 256, + "crop_size": 224, + "interpolation": "bicubic", + "load_model_ema": true, + "use_timm": true, + "timm_model_name": "deit3_small_patch16_224", + "is_vit": true, + "epoch": 300, + "output_head": null, + "model_commitment": { + "layers": [ + "blocks.2.norm1", + "blocks.5.norm1", + "blocks.6.norm2", + "blocks.9.norm2" + ], + "behavioral_readout_layer": "fc_norm", + "region_layer_map": { + "V1": "blocks.2.norm1", + "V2": "blocks.6.norm2", + "V4": "blocks.5.norm1", + "IT": "blocks.9.norm2" + } + }, + "checkpoint_url": "https://epfl-neuroailab-scalinglaws.s3.eu-north-1.amazonaws.com/checkpoints/deit_small_imagenet_full_seed-0/ep300.pt" + }, + "deit_base_imagenet_full_seed-0": { + "model_name": "deit_base", + "model_id": "deit_base_imagenet_full_seed-0", + "num_classes": 1000, + "resize_size": 256, + "crop_size": 224, + "interpolation": "bicubic", + "load_model_ema": true, + "use_timm": true, + "timm_model_name": "deit3_base_patch16_224", + "is_vit": true, + "epoch": 300, + "output_head": null, + "model_commitment": { + "layers": [ + "blocks.3.mlp.act", + "blocks.3.mlp.fc1", + "blocks.8.norm2", + "blocks.9.norm2" + ], + "behavioral_readout_layer": "fc_norm", + "region_layer_map": { + "V1": "blocks.3.mlp.fc1", + "V2": "blocks.8.norm2", + "V4": "blocks.3.mlp.act", + "IT": "blocks.9.norm2" + } + }, + "checkpoint_url": "https://epfl-neuroailab-scalinglaws.s3.eu-north-1.amazonaws.com/checkpoints/deit_base_imagenet_full_seed-0/ep300.pt" + }, + "deit_large_imagenet_full_seed-0": { + "model_name": "deit_large", + "model_id": "deit_large_imagenet_full_seed-0", + "num_classes": 1000, + "resize_size": 256, + "crop_size": 224, + "interpolation": "bicubic", + "load_model_ema": true, + "use_timm": true, + "timm_model_name": "deit3_large_patch16_224", + "is_vit": true, + "epoch": 300, + "output_head": null, + "model_commitment": { + "layers": [ + "blocks.18.norm2", + "blocks.20.norm2", + "blocks.4.norm1", + "blocks.9.norm1" + ], + "behavioral_readout_layer": "fc_norm", + "region_layer_map": { + "V1": "blocks.4.norm1", + "V2": "blocks.18.norm2", + "V4": "blocks.9.norm1", + "IT": "blocks.20.norm2" + } + }, + "checkpoint_url": "https://epfl-neuroailab-scalinglaws.s3.eu-north-1.amazonaws.com/checkpoints/deit_large_imagenet_full_seed-0/ep300.pt" + }, + "deit_small_imagenet_1_seed-0": { + "model_name": "deit_small", + "model_id": "deit_small_imagenet_1_seed-0", + "num_classes": 1000, + "resize_size": 256, + "crop_size": 224, + "interpolation": "bicubic", + "load_model_ema": true, + "use_timm": true, + "timm_model_name": "deit3_small_patch16_224", + "is_vit": true, + "epoch": 300, + "output_head": null, + "model_commitment": { + "layers": [ + "blocks.2.norm1", + "blocks.5.norm1", + "blocks.6.norm2", + "blocks.9.norm2" + ], + "behavioral_readout_layer": "fc_norm", + "region_layer_map": { + "V1": "blocks.2.norm1", + "V2": "blocks.6.norm2", + "V4": "blocks.5.norm1", + "IT": "blocks.9.norm2" + } + }, + "checkpoint_url": "https://epfl-neuroailab-scalinglaws.s3.eu-north-1.amazonaws.com/checkpoints/deit_small_imagenet_1_seed-0/ep300.pt" + }, + "deit_small_imagenet_10_seed-0": { + "model_name": "deit_small", + "model_id": "deit_small_imagenet_10_seed-0", + "num_classes": 1000, + "resize_size": 256, + "crop_size": 224, + "interpolation": "bicubic", + "load_model_ema": true, + "use_timm": true, + "timm_model_name": "deit3_small_patch16_224", + "is_vit": true, + "epoch": 300, + "output_head": null, + "model_commitment": { + "layers": [ + "blocks.2.norm1", + "blocks.5.norm1", + "blocks.6.norm2", + "blocks.9.norm2" + ], + "behavioral_readout_layer": "fc_norm", + "region_layer_map": { + "V1": "blocks.2.norm1", + "V2": "blocks.6.norm2", + "V4": "blocks.5.norm1", + "IT": "blocks.9.norm2" + } + }, + "checkpoint_url": "https://epfl-neuroailab-scalinglaws.s3.eu-north-1.amazonaws.com/checkpoints/deit_small_imagenet_10_seed-0/ep300.pt" + }, + "deit_small_imagenet_100_seed-0": { + "model_name": "deit_small", + "model_id": "deit_small_imagenet_100_seed-0", + "num_classes": 1000, + "resize_size": 256, + "crop_size": 224, + "interpolation": "bicubic", + "load_model_ema": true, + "use_timm": true, + "timm_model_name": "deit3_small_patch16_224", + "is_vit": true, + "epoch": 300, + "output_head": null, + "model_commitment": { + "layers": [ + "blocks.2.norm1", + "blocks.5.norm1", + "blocks.6.norm2", + "blocks.9.norm2" + ], + "behavioral_readout_layer": "fc_norm", + "region_layer_map": { + "V1": "blocks.2.norm1", + "V2": "blocks.6.norm2", + "V4": "blocks.5.norm1", + "IT": "blocks.9.norm2" + } + }, + "checkpoint_url": "https://epfl-neuroailab-scalinglaws.s3.eu-north-1.amazonaws.com/checkpoints/deit_small_imagenet_100_seed-0/ep300.pt" + }, + "convnext_tiny_imagenet_full_seed-0": { + "model_name": "convnext_tiny", + "model_id": "convnext_tiny_imagenet_full_seed-0", + "num_classes": 1000, + "resize_size": 256, + "crop_size": 224, + "interpolation": "bicubic", + "load_model_ema": true, + "use_timm": false, + "timm_model_name": null, + "is_vit": false, + "epoch": 300, + "output_head": "model.classifier[2]", + "model_commitment": { + "layers": [ + "features.4.0", + "features.5.4.block.0", + "features.6.0" + ], + "behavioral_readout_layer": "classifier.1", + "region_layer_map": { + "V1": "features.6.0", + "V2": "features.5.4.block.0", + "V4": "features.4.0", + "IT": "features.5.4.block.0" + } + }, + "checkpoint_url": "https://epfl-neuroailab-scalinglaws.s3.eu-north-1.amazonaws.com/checkpoints/convnext_tiny_imagenet_full_seed-0/ep300.pt" + }, + "convnext_small_imagenet_full_seed-0": { + "model_name": "convnext_small", + "model_id": "convnext_small_imagenet_full_seed-0", + "num_classes": 1000, + "resize_size": 256, + "crop_size": 224, + "interpolation": "bicubic", + "load_model_ema": true, + "use_timm": false, + "timm_model_name": null, + "is_vit": false, + "epoch": 300, + "output_head": "model.classifier[2]", + "model_commitment": { + "layers": [ + "features.4.0", + "features.5.17.block.0", + "features.5.2.block.0", + "features.5.9.block.0" + ], + "behavioral_readout_layer": "classifier.1", + "region_layer_map": { + "V1": "features.5.2.block.0", + "V2": "features.5.17.block.0", + "V4": "features.4.0", + "IT": "features.5.9.block.0" + } + }, + "checkpoint_url": "https://epfl-neuroailab-scalinglaws.s3.eu-north-1.amazonaws.com/checkpoints/convnext_small_imagenet_full_seed-0/ep300.pt" + }, + "convnext_base_imagenet_full_seed-0": { + "model_name": "convnext_base", + "model_id": "convnext_base_imagenet_full_seed-0", + "num_classes": 1000, + "resize_size": 256, + "crop_size": 224, + "interpolation": "bicubic", + "load_model_ema": true, + "use_timm": false, + "timm_model_name": null, + "is_vit": false, + "epoch": 300, + "output_head": "model.classifier[2]", + "model_commitment": { + "layers": [ + "features.4.0", + "features.5.11.block.0", + "features.5.12.block.0", + "features.5.7.block.0" + ], + "behavioral_readout_layer": "classifier.1", + "region_layer_map": { + "V1": "features.5.7.block.0", + "V2": "features.5.12.block.0", + "V4": "features.4.0", + "IT": "features.5.11.block.0" + } + }, + "checkpoint_url": "https://epfl-neuroailab-scalinglaws.s3.eu-north-1.amazonaws.com/checkpoints/convnext_base_imagenet_full_seed-0/ep300.pt" + }, + "convnext_large_imagenet_full_seed-0": { + "model_name": "convnext_large", + "model_id": "convnext_large_imagenet_full_seed-0", + "num_classes": 1000, + "resize_size": 256, + "crop_size": 224, + "interpolation": "bicubic", + "load_model_ema": true, + "use_timm": false, + "timm_model_name": null, + "is_vit": false, + "epoch": 300, + "output_head": "model.classifier[2]", + "model_commitment": { + "layers": [ + "features.4.1", + "features.5.11.block.0", + "features.5.7.block.0", + "features.5.7.block.5" + ], + "behavioral_readout_layer": "classifier.1", + "region_layer_map": { + "V1": "features.5.7.block.5", + "V2": "features.5.7.block.0", + "V4": "features.4.1", + "IT": "features.5.11.block.0" + } + }, + "checkpoint_url": "https://epfl-neuroailab-scalinglaws.s3.eu-north-1.amazonaws.com/checkpoints/convnext_large_imagenet_full_seed-0/ep300.pt" + }, + "convnext_small_imagenet_1_seed-0": { + "model_name": "convnext_small", + "model_id": "convnext_small_imagenet_1_seed-0", + "num_classes": 1000, + "resize_size": 256, + "crop_size": 224, + "interpolation": "bicubic", + "load_model_ema": true, + "use_timm": false, + "timm_model_name": null, + "is_vit": false, + "epoch": 300, + "output_head": "model.classifier[2]", + "model_commitment": { + "layers": [ + "features.4.0", + "features.5.17.block.0", + "features.5.2.block.0", + "features.5.9.block.0" + ], + "behavioral_readout_layer": "classifier.1", + "region_layer_map": { + "V1": "features.5.2.block.0", + "V2": "features.5.17.block.0", + "V4": "features.4.0", + "IT": "features.5.9.block.0" + } + }, + "checkpoint_url": "https://epfl-neuroailab-scalinglaws.s3.eu-north-1.amazonaws.com/checkpoints/convnext_small_imagenet_1_seed-0/ep300.pt" + }, + "convnext_small_imagenet_10_seed-0": { + "model_name": "convnext_small", + "model_id": "convnext_small_imagenet_10_seed-0", + "num_classes": 1000, + "resize_size": 256, + "crop_size": 224, + "interpolation": "bicubic", + "load_model_ema": true, + "use_timm": false, + "timm_model_name": null, + "is_vit": false, + "epoch": 300, + "output_head": "model.classifier[2]", + "model_commitment": { + "layers": [ + "features.4.0", + "features.5.17.block.0", + "features.5.2.block.0", + "features.5.9.block.0" + ], + "behavioral_readout_layer": "classifier.1", + "region_layer_map": { + "V1": "features.5.2.block.0", + "V2": "features.5.17.block.0", + "V4": "features.4.0", + "IT": "features.5.9.block.0" + } + }, + "checkpoint_url": "https://epfl-neuroailab-scalinglaws.s3.eu-north-1.amazonaws.com/checkpoints/convnext_small_imagenet_10_seed-0/ep300.pt" + }, + "convnext_small_imagenet_100_seed-0": { + "model_name": "convnext_small", + "model_id": "convnext_small_imagenet_100_seed-0", + "num_classes": 1000, + "resize_size": 256, + "crop_size": 224, + "interpolation": "bicubic", + "load_model_ema": true, + "use_timm": false, + "timm_model_name": null, + "is_vit": false, + "epoch": 300, + "output_head": "model.classifier[2]", + "model_commitment": { + "layers": [ + "features.4.0", + "features.5.17.block.0", + "features.5.2.block.0", + "features.5.9.block.0" + ], + "behavioral_readout_layer": "classifier.1", + "region_layer_map": { + "V1": "features.5.2.block.0", + "V2": "features.5.17.block.0", + "V4": "features.4.0", + "IT": "features.5.9.block.0" + } + }, + "checkpoint_url": "https://epfl-neuroailab-scalinglaws.s3.eu-north-1.amazonaws.com/checkpoints/convnext_small_imagenet_100_seed-0/ep300.pt" + } +} \ No newline at end of file diff --git a/brainscore_vision/models/scaling_models/requirements.txt b/brainscore_vision/models/scaling_models/requirements.txt new file mode 100644 index 000000000..af6389b09 --- /dev/null +++ b/brainscore_vision/models/scaling_models/requirements.txt @@ -0,0 +1,4 @@ +torch +torchvision +albumentations +timm diff --git a/brainscore_vision/models/scaling_models/test.py b/brainscore_vision/models/scaling_models/test.py new file mode 100644 index 000000000..e69de29bb