searchindex.js

Search.setIndex({"docnames": ["api", "call_for_papers", "index", "papers", "schedule"], "filenames": ["api.rst", "call_for_papers.rst", "index.rst", "papers.rst", "schedule.rst"], "titles": ["Welcome to the API docs of the ShiftHappens benchmark!", "Call for Submissions", "Shift happens: Crowdsourcing metrics and test datasets beyond ImageNet", "Accepted Papers", "ShiftHappens Workshop Schedule"], "terms": {"we": [0, 1, 2], "aim": [0, 2], "creat": [0, 1, 2], "commun": [0, 1, 2, 4], "built": [0, 2], "suit": [0, 1, 2], "compris": [0, 1, 2], "new": [0, 1, 2, 3], "dataset": [0, 1, 4], "ood": [0, 1, 2, 3], "robust": [0, 1, 2, 3, 4], "detect": [0, 1, 2], "well": [0, 1, 2, 3], "exist": [0, 1, 2], "while": [0, 1, 2], "popular": [0, 1, 2], "test": [0, 1], "increas": [0, 2], "over": [0, 1, 2, 3], "past": [0, 2], "year": [0, 2], "perform": [0, 1, 2], "comput": [0, 2, 4], "vision": [0, 2, 4], "i": [0, 1, 2], "still": [0, 2, 3], "larg": [0, 1, 2], "evalu": [0, 2, 3, 4], "directli": [0, 1, 2], "simul": [0, 2], "isol": [0, 2], "distribut": [0, 1, 2, 3, 4], "shift": [0, 1, 3, 4], "like": [0, 1, 2], "c": [0, 1, 2], "The": [0, 1, 2, 3, 4], "goal": [0, 2], "thi": [0, 1, 2], "workshop": [0, 1, 2], "enhanc": [0, 2], "landscap": [0, 2], "devis": 0, "metric": [0, 1], "quantifi": [0, 2], "desir": [0, 2], "properti": [0, 1, 2], "our": [0, 1, 2], "bring": [0, 2], "domain": [0, 1, 2, 3], "adapt": [0, 2, 3], "out": [0, 1, 2, 3, 4], "togeth": [0, 2], "work": [0, 1, 2], "broad": [0, 1, 2], "scale": [0, 1, 2], "divers": [0, 2], "aspect": [0, 2], "current": [0, 1, 2], "guid": [0, 2], "wai": [0, 2], "toward": [0, 1, 2, 3], "next": [0, 2], "gener": [0, 1, 2, 4], "submiss": 0, "an": [0, 1, 2], "addit": [0, 1, 2], "which": [0, 1, 2], "us": [0, 1, 2], "variou": 0, "you": [0, 1, 2], "specifi": [0, 1], "your": [0, 1], "below": [0, 1, 2], "provid": [0, 1, 2], "document": [0, 2], "also": [0, 1, 2, 4], "make": [0, 1, 2], "sure": [0, 1, 2], "look": 0, "exampl": [0, 1], "github": [0, 2], "repositori": 0, "If": [0, 1], "doubt": 0, "yet": [0, 1], "suffici": [0, 1, 2], "flexibl": 0, "fit": 0, "need": [0, 1], "consid": 0, "open": [0, 2, 4], "issu": 0, "join": [0, 1, 2], "slack": [0, 1, 2], "channel": [0, 1, 2], "base": [0, 1], "function": 0, "regist": 0, "To": [0, 2], "add": 0, "decor": 0, "class": [0, 1, 2, 3], "inherit": 0, "register_task": 0, "all": [0, 1, 2, 4], "run": [0, 1], "evaluate_model": 0, "data_root": 0, "sourc": [0, 2], "ar": [0, 1, 2, 4], "support": 0, "suppli": 0, "paramet": 0, "str": 0, "folder": 0, "where": [0, 2], "individu": [0, 3], "can": [0, 1, 2], "return": 0, "type": 0, "dict": 0, "taskregistr": 0, "option": [0, 2], "taskresult": 0, "associ": [0, 1], "task_data": 0, "task_metadata": 0, "taskmetadata": 0, "respect": [0, 2], "task_result": 0, "import": 0, "resnet18": 0, "custom": 0, "one": [0, 1, 2], "modelmixin": 0, "": [0, 2], "path_to_store_tasks_data": 0, "name": 0, "relative_data_fold": 0, "standalon": 0, "true": 0, "part": [0, 1, 2], "contain": [0, 1, 2], "space": 0, "special": [0, 1, 2], "charact": 0, "save": 0, "rel": 0, "root": 0, "bool": 0, "default": 0, "boolean": 0, "repres": 0, "meaning": 0, "onli": [0, 1], "relev": [0, 1, 2], "collect": [0, 1, 2], "customtask": 0, "path_to_store_task_data": 0, "dataclass": 0, "customtaskclass": 0, "get_registered_task": 0, "tupl": 0, "A": [0, 1, 2, 3, 4], "iter": 0, "definit": [0, 1], "happen": 0, "fulli": 0, "defin": 0, "should": [0, 1, 3], "subclass": 0, "abstract": [0, 1, 2], "mixin": 0, "requir": [0, 1, 2, 4], "output": [0, 1], "modul": 0, "consist": [0, 1, 2], "follow": [0, 1], "step": 0, "its": [0, 1], "method": [0, 2], "setup": [0, 1], "scheme": 0, "ani": [0, 1, 2], "number": [0, 1], "just": 0, "includ": [0, 1, 2], "e": [0, 1], "g": [0, 1, 2], "mytask": 0, "confidencetaskmixin": 0, "do": [0, 1, 2, 3], "By": 0, "assur": 0, "get": [0, 1, 2], "correct": [0, 1], "see": [0, 1, 2], "modelresult": 0, "further": [0, 1], "detail": [0, 2], "along": [0, 1], "path": 0, "t": [0, 1], "arbitrari": 0, "alia": 0, "typevar": 0, "descript": [0, 1], "none": 0, "multipl": [0, 1], "here": [0, 1, 2], "allow": [0, 1], "automat": 0, "differ": [0, 1, 2], "flavour": 0, "field": 0, "valu": [0, 1], "hyperparamet": 0, "want": [0, 2], "For": [0, 1, 2], "mechan": [0, 2], "vari": 0, "difficulti": [0, 1, 3], "without": [0, 1], "short": [0, 1], "max_batch_s": 0, "int": 0, "cast": 0, "32": 0, "64": 0, "128": 0, "correspond": [0, 1, 2], "size": 0, "batch": 0, "maximum": 0, "fed": 0, "dure": [0, 1], "variabl": 0, "non": [0, 1, 2], "parametr": 0, "won": 0, "pass": 0, "constructor": 0, "constant": 0, "link": [0, 1], "abstract_vari": 0, "mark": 0, "child": 0, "overrid": 0, "counterpart": 0, "inheritedtask": 0, "abc": 0, "_prepare_dataload": 0, "_evalu": 0, "compat": [0, 1], "label": [0, 1, 2], "confid": 0, "initi": 0, "classmethod": 0, "iterate_flavour": 0, "kwarg": 0, "possibl": [0, 1], "argument": 0, "particular": [0, 1], "up": [0, 2], "download": [0, 1], "prepar": [0, 1, 2], "imagenet_r": 0, "imagenetr": 0, "def": 0, "self": 0, "dataset_fold": 0, "o": 0, "r": [0, 1, 3], "file_nam": 0, "url": 0, "md5": 0, "resourc": 0, "sh_util": 0, "download_and_extract_arch": 0, "more": [0, 1, 2], "dataload": 0, "unlabel": [0, 2], "imag": [0, 1, 2, 3], "befor": [0, 2], "actual": 0, "unsupervis": 0, "techniqu": [0, 2], "give": [0, 1], "case": [0, 1], "thei": [0, 1], "some": [0, 1], "kind": 0, "re": [0, 1], "calibr": [0, 1, 2], "note": [0, 1, 2], "could": [0, 1], "introduc": 0, "than": 0, "dictionari": 0, "calcul": [0, 1], "predict": [0, 1, 2], "each": 0, "result": [0, 2], "assign": 0, "etc": [0, 1], "were": 0, "request": 0, "form": [0, 1, 2], "specif": [0, 1], "main": 0, "summari": [0, 1], "all_predicted_labels_list": 0, "predictiontarget": 0, "class_label": 0, "append": 0, "all_predicted_label": 0, "np": 0, "concaten": 0, "0": [0, 4], "accuraci": [0, 1], "arrai": 0, "ch_dataset": 0, "target": 0, "summary_metr": 0, "indic": 0, "term": 0, "labeltaskmixin": 0, "object": [0, 1, 2, 3], "attribut": 0, "uncertaintytaskmixin": 0, "uncertainti": [0, 1, 2], "oodscoretaskmixin": 0, "ood_scor": 0, "featurestaskmixin": 0, "raw": 0, "featur": [0, 1], "singl": [0, 1], "At": 0, "least": [0, 1], "must": 0, "refer": [0, 1], "union": 0, "float": 0, "your_robustness_metr": 0, "your_calibration_metr": 0, "your_custom_metr": 0, "1": [0, 1, 2, 4], "categori": [0, 2], "group": [0, 2], "intenum": 0, "aggreg": 0, "averag": [0, 1], "global": 0, "config": 0, "imagenet_validation_path": 0, "imagenet21k_preprocessed_validation_path": 0, "imagenet21k": 0, "p": 0, "cache_directory_path": 0, "cach": 0, "verbos": 0, "fals": 0, "edit": 0, "order": 0, "explicitli": 0, "when": [0, 3, 4], "instanc": 0, "via": [0, 1], "get_config": 0, "environ": 0, "prefix": 0, "sh_variable_nam": 0, "reli": 0, "directori": 0, "either": 0, "absolut": 0, "doe": [0, 3, 4], "show": 0, "log": 0, "messag": 0, "stderr": 0, "progress": 0, "bar": 0, "get_inst": 0, "init_kwarg": 0, "chang": [0, 3], "reiniti": 0, "rais": 0, "error": 0, "reduc": 0, "cost": 0, "pleas": [0, 1, 2], "have": [0, 1, 2], "py": 0, "get_imagenet_validation_load": 0, "how": [0, 1], "mani": 0, "sampl": [0, 1], "per": 0, "loader": 0, "get_cached_predict": 0, "cl": 0, "check": [0, 1], "whether": 0, "so": [0, 1], "them": [0, 1], "cache_predict": 0, "imagenet_validation_result": 0, "is_cach": 0, "otherwis": 0, "load_imagenet_target": 0, "ground": 0, "truth": 0, "valdat": 0, "ndarrai": 0, "helper": 0, "ad": [0, 1, 3], "wrapper": 0, "convert": 0, "numpi": 0, "pack": 0, "emiss": 0, "after": [0, 1, 2], "process": [0, 1], "ilsrc2012": 0, "same": 0, "convent": [0, 4], "pytorch": 0, "regard": 0, "n": 0, "k": 0, "top": [0, 1], "choic": 0, "select": 0, "user": 0, "potenti": [0, 1], "influenc": [0, 2], "standard": [0, 1, 2], "choos": 0, "5": [0, 2, 4], "1000": [0, 2], "expect": 0, "rang": 0, "inf": 0, "measur": [0, 3], "certainti": 0, "given": 0, "common": [0, 3], "bayesian": 0, "deep": 0, "neural": 0, "network": 0, "interpret": [0, 1], "d": [0, 3], "represent": [0, 2], "arriv": 0, "flag": 0, "_predict": 0, "labelmodelmixin": 0, "confidencemodelmixin": 0, "wrap": 0, "those": 0, "produc": 0, "input_dataload": 0, "yield": 0, "depend": 0, "_get_imagenet_predict": 0, "rewrit": 0, "whenev": 0, "rewritten": 0, "_predict_imagenet_v": 0, "mai": [0, 1, 2], "uncertaintymodelmixin": 0, "oodscoremodelmixin": 0, "featuresmodelmixin": 0, "baselin": 0, "__torchvisionpreprocessingmixin": 0, "preprocess": 0, "classifi": [0, 3], "__torchvisionmodel": 0, "feature_lay": 0, "devic": 0, "cpu": 0, "pretrain": 0, "layer": 0, "shall": 0, "extract": 0, "16": [0, 4], "train": [0, 1], "2012": [0, 2], "param": 0, "resnet50": [0, 1], "vgg16": 0, "handl": 0, "indexeddataset": 0, "map": 0, "style": [0, 1], "interfac": 0, "b": 0, "w": [0, 1], "restrict": 0, "max": 0, "satisfi": 0, "through": [0, 2], "batch_siz": 0, "split": 0, "list": 0, "shuffle_data": 0, "seed": 0, "randomli": 0, "shuffl": 0, "replac": 0, "fix": 0, "random": [0, 1], "torch": 0, "torchdataset": 0, "torch_dataset": 0, "util": [0, 2], "iterabledataset": 0, "indexedtorchdataset": 0, "imagesonlytorchdataset": 0, "metadata": 0, "registr": [0, 2, 4], "task_registr": 0, "now": [1, 2], "close": [1, 2, 4], "updat": 1, "camera": [1, 2], "readi": [1, 2], "templat": 1, "accept": [1, 2], "full": [1, 2], "extend": [1, 2], "latex": [1, 2], "modifi": [1, 2], "from": [1, 2], "offici": 1, "icml": [1, 2], "paper": [1, 2], "found": [1, 2], "technic": 1, "task": [1, 2], "implement": 1, "extern": 1, "file": [1, 2], "both": 1, "compon": 1, "submit": 1, "openreview": [1, 2], "review": [1, 2], "doubl": 1, "blind": 1, "time": [1, 2, 3, 4], "proof": 1, "concept": 1, "until": [1, 2], "plugin": 1, "shifthappen": [1, 2], "benchmark": [1, 2, 4], "packag": [1, 2], "final": [1, 2], "version": 1, "two": [1, 3], "page": [1, 2], "pdf": 1, "figur": 1, "In": [1, 2], "author": [1, 2], "describ": 1, "idea": [1, 2], "interest": [1, 2], "intend": 1, "futur": [1, 2], "peer": 1, "feedback": 1, "proceed": 1, "besid": 1, "archiv": 1, "been": [1, 2], "recent": 1, "public": 1, "encourag": [1, 2], "contribut": [1, 2, 4], "classif": 1, "imagenet": [1, 3, 4], "fair": 1, "complet": [1, 2], "highlight": [1, 2], "between": 1, "model": [1, 2, 3], "problem": 1, "focus": [1, 2], "set": [1, 2], "veri": 1, "welcom": 1, "question": [1, 2], "about": [1, 2], "clarif": [1, 2], "around": [1, 2], "score": [1, 2, 3], "discuss": [1, 2, 4], "api": [1, 2], "feel": [1, 2], "free": [1, 2], "write": [1, 2], "u": [1, 2], "bethgelab": [1, 2], "org": [1, 2], "find": [1, 2], "june": [1, 2], "9": [1, 2], "2022": [1, 2, 4], "previous": [1, 2], "3": [1, 2, 4], "27": [1, 2], "neurip": [1, 2], "track": [1, 2], "post": [1, 2], "13": [1, 2, 4], "notif": [1, 2], "juli": [1, 2, 4], "date": [1, 2], "22": [1, 2], "It": [1, 2, 4], "start": [1, 2, 4], "long": 1, "demonstr": 1, "applic": 1, "activ": 1, "integr": 1, "supplementari": 1, "materi": 1, "2": [1, 4], "4": [1, 4], "later": 1, "host": 1, "scope": 1, "intention": 1, "own": [1, 2], "criterion": 1, "particularli": 1, "explain": 1, "why": 1, "auroc": 1, "fpr": 1, "aupr": 1, "other": [1, 2], "pre": [1, 2], "There": 1, "constraint": 1, "publish": 1, "consolid": [1, 2], "known": 1, "small": 1, "occur": 1, "due": 1, "real": [1, 3], "world": [1, 2, 3], "insight": 1, "might": 1, "advantag": 1, "certain": 1, "relat": 1, "15": [1, 4], "corrupt": [1, 3], "correspondingli": 1, "demand": 1, "ideal": 1, "fool": 1, "cours": 1, "sens": 1, "multi": 1, "motiv": 1, "wa": [1, 2], "overview": [1, 2], "state": 1, "access": [1, 2], "procedur": 1, "outlin": 1, "analysi": 1, "challeng": [1, 3, 4], "ones": 1, "establish": [1, 2], "context": 1, "within": 1, "limit": 1, "appendix": 1, "reserv": 1, "format": [1, 2], "accord": 1, "capabl": [1, 2], "becom": [1, 3, 4], "clear": 1, "would": 1, "invit": [1, 4], "build": 1, "whatev": 1, "suitabl": 1, "even": [1, 3], "outsid": 1, "hesit": 1, "reach": 1, "email": 1, "continu": [1, 2, 3], "assist": [1, 2], "platform": 1, "zenodo": 1, "googl": [1, 2], "drive": 1, "anonym": 1, "id": 1, "usabl": 1, "research": [1, 2], "purpos": 1, "Their": [1, 3], "licens": 1, "doc": 1, "judg": 1, "human": 1, "visibl": 1, "worst": [1, 3], "mistak": [1, 3, 4], "mainli": 1, "novelti": 1, "beyond": 1, "emphas": 1, "practic": [1, 2], "theoret": [1, 2], "weak": [1, 2], "easili": 1, "solv": 1, "fail": 1, "moder": 1, "spectacularli": 1, "formal": 1, "robustifi": 1, "problemat": 2, "odd": 2, "intuit": 2, "scorecard": 2, "ensur": 2, "central": 2, "facilit": 2, "fine": 2, "grain": 2, "inform": 2, "card": 2, "credit": 2, "effort": 2, "releas": 2, "bibtex": 2, "cite": 2, "macro": 2, "underli": 2, "particip": 2, "opportun": 2, "co": 2, "summar": 2, "algorithm": 2, "fall": 2, "subset": 2, "especi": 2, "covari": 2, "enabl": 2, "spuriou": 2, "correl": 2, "characterist": 2, "help": 2, "examin": 2, "strength": 2, "peculiar": 2, "newli": 2, "thereof": 2, "come": 2, "avail": 2, "call": 2, "sinc": 2, "feasibl": 2, "offer": 2, "outstand": 2, "aleksand": [2, 4], "m\u0105dry": [2, 4], "cadenc": 2, "design": 2, "system": 2, "professor": 2, "mit": 2, "lead": 2, "center": [2, 4], "deploy": 2, "machin": [2, 3, 4], "learn": [2, 3, 4], "faculti": 2, "ai": 2, "polici": 2, "forum": 2, "hi": 2, "span": 2, "optim": 2, "understand": 2, "perspect": 2, "alexei": [2, 4], "efro": [2, 4], "eec": 2, "depart": 2, "uc": 2, "berkelei": 2, "he": [2, 3], "artifici": 2, "intellig": 2, "lab": 2, "bair": 2, "spent": 2, "nine": 2, "robot": 2, "institut": 2, "cmu": 2, "2007": 2, "collabor": 2, "team": 2, "willow": 2, "\u00e9cole": 2, "normal": 2, "sup\u00e9rieur": 2, "inria": 2, "pari": 2, "vast": 2, "amount": 2, "visual": [2, 3], "data": 2, "recreat": 2, "chelsea": [2, 3, 4], "finn": [2, 3, 4], "scienc": 2, "electr": 2, "engin": 2, "stanford": 2, "univers": 2, "her": 2, "iri": 2, "studi": 2, "interact": 2, "affili": 2, "sail": 2, "ml": 2, "spend": 2, "brain": 2, "she": 2, "agent": 2, "develop": 2, "broadli": 2, "behavior": 2, "luca": [2, 3, 4], "beyer": [2, 3, 4], "grew": 2, "belgium": 2, "video": 2, "game": 2, "went": 2, "rwth": 2, "aachen": 2, "germani": 2, "did": 2, "phd": 2, "percept": 2, "too": 2, "backbon": 2, "z\u00fcrich": 2, "ludwig": [2, 4], "schmidt": [2, 4], "paul": 2, "allen": 2, "school": 2, "washington": 2, "revolv": 2, "empir": 2, "foundat": 2, "often": 2, "reliabl": 2, "under": [2, 3], "supervis": 2, "piotr": 2, "indyk": 2, "postdoc": 2, "benjamin": [2, 3], "recht": 2, "moritz": 2, "hardt": 2, "receiv": 2, "fellowship": 2, "microsoft": 2, "simon": 2, "horizon": 2, "award": 2, "eaamo": 2, "best": 2, "sprowl": 2, "dissert": 2, "julian": [2, 3], "bitterwolf": [2, 3], "evgenia": [2, 3], "rusak": [2, 3], "steffen": [2, 3], "schneider": [2, 3], "roland": 2, "zimmermann": 2, "matthia": [2, 3], "bethg": [2, 3], "wieland": [2, 3], "brendel": [2, 3], "hein": [2, 3], "thank": 2, "lot": 2, "shape": 2, "akshai": 2, "ravindra": 2, "kulkarni": 2, "alexand": [2, 3], "meink": [2, 3], "ilia": 2, "sucholutski": 2, "leander": [2, 3], "kurscheidt": [2, 3], "maura": [2, 3], "pintor": [2, 3], "mert": 2, "yuksekgonul": 2, "ori": [2, 3], "press": [2, 3], "prasanna": 2, "mayilvahanan": 2, "rebecca": [2, 3], "roelof": [2, 3], "teja": 2, "gokhal": 2, "valentyn": [2, 3], "boreiko": [2, 3], "victor": 2, "bouvier": 2, "zhiheng": 2, "li": 2, "patch": [3, 4], "against": [3, 4], "adversari": [3, 4], "daniel": 3, "angioni": 3, "angelo": 3, "sotgiu": 3, "demetrio": 3, "ambra": 3, "demonti": 3, "battista": 3, "biggio": 3, "fabio": 3, "roli": 3, "metashift": [3, 4], "contextu": [3, 4], "weixin": 3, "liang": 3, "xinyu": 3, "yang": 3, "jame": 3, "y": 3, "zou": 3, "dough": [3, 4], "bagel": [3, 4], "analyz": [3, 4], "remain": [3, 4], "vijai": 3, "vasudevan": 3, "cain": 3, "raphael": 3, "gontijo": 3, "lope": 3, "sara": 3, "fridovich": 3, "keil": 3, "3d": 3, "recognit": 3, "oguzhan": 3, "fatih": 3, "kar": 3, "teresa": 3, "yeo": 3, "amir": 3, "zamir": 3, "ccc": 3, "kuemmer": 3, "perturb": 3, "jindong": 3, "gu": 3, "volker": 3, "tresp": 3, "yao": 3, "qin": 3, "cartoon": 3, "draw": 3, "tiago": 3, "salvador": 3, "adam": 3, "m": 3, "oberman": 3, "inspir": 3, "peter": 3, "vincent": 3, "gehler": 3, "oliv": 3, "bringmann": 3, "lost": 3, "translat": 3, "modern": 3, "degrad": 3, "simpl": 3, "si": 3, "jessica": 3, "yung": 3, "rob": 3, "romijnd": 3, "kolesnikov": 3, "josip": 3, "djolonga": 3, "neil": 3, "houlsbi": 3, "sylvain": 3, "gelli": 3, "mario": 3, "lucic": 3, "xiaohua": 3, "zhai": 3, "semant": 3, "sagar": 3, "vaze": 3, "kai": 3, "han": 3, "andrea": 3, "vedaldi": 3, "andrew": 3, "zisserman": 3, "grow": 3, "objectnet": 3, "speech": 3, "vqa": 3, "occlus": 3, "david": 3, "mayo": 3, "lu": 3, "chri": 3, "zhang": 3, "jess": 3, "cum": 3, "lin": 3, "bori": 3, "katz": 3, "glass": 3, "andrei": 3, "barbu": 3, "cv": 3, "nuisanc": 3, "bingchen": 3, "zhao": 3, "shaozuo": 3, "yu": 3, "wufei": 3, "ma": 3, "mingxin": 3, "shenxiao": 3, "mei": 3, "angtian": 3, "wang": 3, "ju": 3, "alan": 3, "yuill": 3, "kortylewski": 3, "systemat": 3, "scalabl": 3, "moham": 3, "omran": 3, "bernt": 3, "schiel": 3, "wild": 3, "huaxiu": 3, "carolin": 3, "choi": 3, "yoonho": 3, "lee": 3, "pang": 3, "wei": 3, "koh": 3, "take": 4, "place": 4, "22nd": 4, "ballroom": 4, "baltimor": 4, "stream": 4, "onlin": 4, "local": 4, "utc": 4, "end": 4, "09": 4, "00": 4, "10": 4, "introduct": 4, "remark": 4, "25": 4, "talk": 4, "05": 4, "35": 4, "coffe": 4, "break": 4, "50": 4, "11": 4, "30": 4, "12": 4, "lunch": 4, "14": 4, "lightn": 4, "poster": 4, "session": 4, "20": 4, "tea": 4, "17": 4, "18": 4, "panel": 4, "soheil": 4, "feizi": 4, "present": 4, "45": 4, "19": 4}, "objects": {"shifthappens": [[0, 0, 0, "-", "benchmark"], [0, 0, 0, "-", "config"]], "shifthappens.benchmark": [[0, 1, 1, "", "evaluate_model"], [0, 1, 1, "", "get_registered_tasks"], [0, 1, 1, "", "register_task"]], "shifthappens.config": [[0, 2, 1, "", "Config"], [0, 1, 1, "", "get_config"]], "shifthappens.config.Config": [[0, 3, 1, "", "cache_directory_path"], [0, 4, 1, "", "get_instance"], [0, 3, 1, "", "imagenet21k_preprocessed_validation_path"], [0, 3, 1, "", "imagenet_validation_path"], [0, 3, 1, "", "verbose"]], "shifthappens.data": [[0, 0, 0, "-", "base"], [0, 0, 0, "-", "imagenet"], [0, 0, 0, "-", "torch"]], "shifthappens.data.base": [[0, 2, 1, "", "DataLoader"], [0, 2, 1, "", "Dataset"], [0, 2, 1, "", "IndexedDataset"], [0, 1, 1, "", "shuffle_data"]], "shifthappens.data.base.DataLoader": [[0, 4, 1, "", "iterate"], [0, 5, 1, "", "max_batch_size"]], "shifthappens.data.imagenet": [[0, 1, 1, "", "cache_predictions"], [0, 1, 1, "", "get_cached_predictions"], [0, 1, 1, "", "get_imagenet_validation_loader"], [0, 1, 1, "", "is_cached"], [0, 1, 1, "", "load_imagenet_targets"]], "shifthappens.data.torch": [[0, 2, 1, "", "ImagesOnlyTorchDataset"], [0, 2, 1, "", "IndexedTorchDataset"], [0, 2, 1, "", "TorchDataset"]], "shifthappens.models": [[0, 0, 0, "-", "base"], [0, 0, 0, "-", "mixins"], [0, 0, 0, "-", "torchvision"]], "shifthappens.models.base": [[0, 2, 1, "", "Model"], [0, 2, 1, "", "ModelResult"], [0, 2, 1, "", "PredictionTargets"]], "shifthappens.models.base.Model": [[0, 4, 1, "", "_get_imagenet_predictions"], [0, 4, 1, "", "_predict"], [0, 4, 1, "", "_predict_imagenet_val"], [0, 5, 1, "", "imagenet_validation_result"], [0, 4, 1, "", "predict"], [0, 4, 1, "", "prepare"]], "shifthappens.models.mixins": [[0, 2, 1, "", "ConfidenceModelMixin"], [0, 2, 1, "", "FeaturesModelMixin"], [0, 2, 1, "", "LabelModelMixin"], [0, 2, 1, "", "OODScoreModelMixin"], [0, 2, 1, "", "UncertaintyModelMixin"]], "shifthappens.models.torchvision": [[0, 2, 1, "", "ResNet18"], [0, 2, 1, "", "ResNet50"], [0, 2, 1, "", "VGG16"], [0, 2, 1, "", "__TorchvisionModel"], [0, 2, 1, "", "__TorchvisionPreProcessingMixin"]], "shifthappens.models.torchvision.__TorchvisionModel": [[0, 4, 1, "", "_predict"]], "shifthappens.task_data": [[0, 0, 0, "-", "task_metadata"], [0, 0, 0, "-", "task_registration"]], "shifthappens.task_data.task_metadata": [[0, 2, 1, "", "TaskMetadata"]], "shifthappens.task_data.task_registration": [[0, 2, 1, "", "TaskRegistration"]], "shifthappens.tasks": [[0, 0, 0, "-", "base"], [0, 0, 0, "-", "metrics"], [0, 0, 0, "-", "mixins"], [0, 0, 0, "-", "task_result"]], "shifthappens.tasks.base": [[0, 6, 1, "", "T"], [0, 2, 1, "", "Task"], [0, 1, 1, "", "abstract_variable"], [0, 1, 1, "", "parameter"], [0, 1, 1, "", "variable"]], "shifthappens.tasks.base.Task": [[0, 4, 1, "", "_evaluate"], [0, 4, 1, "", "_prepare_dataloader"], [0, 4, 1, "", "evaluate"], [0, 4, 1, "", "iterate_flavours"], [0, 4, 1, "", "setup"]], "shifthappens.tasks.metrics": [[0, 2, 1, "", "Metric"]], "shifthappens.tasks.mixins": [[0, 2, 1, "", "ConfidenceTaskMixin"], [0, 2, 1, "", "FeaturesTaskMixin"], [0, 2, 1, "", "LabelTaskMixin"], [0, 2, 1, "", "OODScoreTaskMixin"], [0, 2, 1, "", "UncertaintyTaskMixin"]], "shifthappens.tasks.task_result": [[0, 2, 1, "", "TaskResult"]]}, "objtypes": {"0": "py:module", "1": "py:function", "2": "py:class", "3": "py:attribute", "4": "py:method", "5": "py:property", "6": "py:data"}, "objnames": {"0": ["py", "module", "Python module"], "1": ["py", "function", "Python function"], "2": ["py", "class", "Python class"], "3": ["py", "attribute", "Python attribute"], "4": ["py", "method", "Python method"], "5": ["py", "property", "Python property"], "6": ["py", "data", "Python data"]}, "titleterms": {"welcom": 0, "api": 0, "doc": 0, "shifthappen": [0, 4], "benchmark": [0, 3], "task": 0, "implement": 0, "configur": 0, "access": 0, "score": 0, "imagenet": [0, 2], "valid": 0, "set": 0, "model": 0, "from": 0, "torchvis": 0, "data": [0, 1], "load": 0, "store": 0, "within": 0, "call": 1, "submiss": [1, 2], "tl": 1, "dr": 1, "deadlin": [1, 2], "detail": 1, "inform": 1, "type": 1, "topic": [1, 2], "report": 1, "instruct": 1, "code": 1, "evalu": 1, "criteria": 1, "shift": 2, "happen": 2, "crowdsourc": 2, "metric": [2, 3], "test": 2, "dataset": [2, 3], "beyond": 2, "focu": 2, "import": 2, "prize": 2, "travel": 2, "grant": 2, "invit": 2, "speaker": 2, "organ": 2, "program": 2, "committe": 2, "accept": 3, "paper": 3, "track": 3, "1": 3, "contribut": 3, "talk": 3, "poster": 3, "2": 3, "extend": 3, "abstract": 3, "workshop": 4, "schedul": 4}, "envversion": {"sphinx.domains.c": 2, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 8, "sphinx.domains.index": 1, "sphinx.domains.javascript": 2, "sphinx.domains.math": 2, "sphinx.domains.python": 3, "sphinx.domains.rst": 2, "sphinx.domains.std": 2, "sphinx.ext.viewcode": 1, "sphinx.ext.intersphinx": 1, "sphinx": 57}, "alltitles": {"Welcome to the API docs of the ShiftHappens benchmark!": [[0, "welcome-to-the-api-docs-of-the-shifthappens-benchmark"]], "Benchmark": [[0, "module-shifthappens.benchmark"]], "Task implementations": [[0, "module-shifthappens.tasks.base"]], "Configuration": [[0, "module-shifthappens.config"]], "Accessing Scores for the ImageNet Validation Set": [[0, "module-shifthappens.data.imagenet"]], "Model implementations": [[0, "module-shifthappens.models.base"]], "Model implementations from torchvision": [[0, "module-shifthappens.models.torchvision"]], "Data loading": [[0, "module-shifthappens.data.base"]], "Storing tasks within the benchmark": [[0, "module-shifthappens.task_data.task_metadata"]], "Call for Submissions": [[1, "call-for-submissions"]], "TL;DR": [[1, "tl-dr"]], "Deadlines": [[1, "deadlines"]], "Detailed Information on Submission Types and Topics": [[1, "detailed-information-on-submission-types-and-topics"]], "Submission Types": [[1, "submission-types"]], "Topics": [[1, "topics"]], "Report Instructions": [[1, "report-instructions"]], "Code and Data Instructions": [[1, "code-and-data-instructions"]], "Evaluation Criteria for Submissions": [[1, "evaluation-criteria-for-submissions"]], "Shift happens: Crowdsourcing metrics and test datasets beyond ImageNet": [[2, "shift-happens-crowdsourcing-metrics-and-test-datasets-beyond-imagenet"]], "Focus Topics": [[2, "focus-topics"]], "Submissions": [[2, "submissions"]], "Important Deadlines": [[2, "important-deadlines"]], "Prizes and Travel Grants": [[2, "prizes-and-travel-grants"]], "Invited Speakers": [[2, "invited-speakers"]], "Organizers": [[2, "organizers"]], "Program Committee": [[2, "program-committee"]], "Accepted Papers": [[3, "accepted-papers"]], "Track 1: Benchmarks, Datasets and Metrics": [[3, "track-1-benchmarks-datasets-and-metrics"]], "Contributed Talks": [[3, "contributed-talks"]], "Posters": [[3, "posters"]], "Track 2: Extended abstracts": [[3, "track-2-extended-abstracts"]], "ShiftHappens Workshop Schedule": [[4, "shifthappens-workshop-schedule"]], "Schedule": [[4, "id1"]]}, "indexentries": {"confidencemodelmixin (class in shifthappens.models.mixins)": [[0, "shifthappens.models.mixins.ConfidenceModelMixin"]], "confidencetaskmixin (class in shifthappens.tasks.mixins)": [[0, "shifthappens.tasks.mixins.ConfidenceTaskMixin"]], "config (class in shifthappens.config)": [[0, "shifthappens.config.Config"]], "dataloader (class in shifthappens.data.base)": [[0, "shifthappens.data.base.DataLoader"]], "dataset (class in shifthappens.data.base)": [[0, "shifthappens.data.base.Dataset"]], "featuresmodelmixin (class in shifthappens.models.mixins)": [[0, "shifthappens.models.mixins.FeaturesModelMixin"]], "featurestaskmixin (class in shifthappens.tasks.mixins)": [[0, "shifthappens.tasks.mixins.FeaturesTaskMixin"]], "imagesonlytorchdataset (class in shifthappens.data.torch)": [[0, "shifthappens.data.torch.ImagesOnlyTorchDataset"]], "indexeddataset (class in shifthappens.data.base)": [[0, "shifthappens.data.base.IndexedDataset"]], "indexedtorchdataset (class in shifthappens.data.torch)": [[0, "shifthappens.data.torch.IndexedTorchDataset"]], "labelmodelmixin (class in shifthappens.models.mixins)": [[0, "shifthappens.models.mixins.LabelModelMixin"]], "labeltaskmixin (class in shifthappens.tasks.mixins)": [[0, "shifthappens.tasks.mixins.LabelTaskMixin"]], "metric (class in shifthappens.tasks.metrics)": [[0, "shifthappens.tasks.metrics.Metric"]], "model (class in shifthappens.models.base)": [[0, "shifthappens.models.base.Model"]], "modelresult (class in shifthappens.models.base)": [[0, "shifthappens.models.base.ModelResult"]], "oodscoremodelmixin (class in shifthappens.models.mixins)": [[0, "shifthappens.models.mixins.OODScoreModelMixin"]], "oodscoretaskmixin (class in shifthappens.tasks.mixins)": [[0, "shifthappens.tasks.mixins.OODScoreTaskMixin"]], "predictiontargets (class in shifthappens.models.base)": [[0, "shifthappens.models.base.PredictionTargets"]], "resnet18 (class in shifthappens.models.torchvision)": [[0, "shifthappens.models.torchvision.ResNet18"]], "resnet50 (class in shifthappens.models.torchvision)": [[0, "shifthappens.models.torchvision.ResNet50"]], "t (in module shifthappens.tasks.base)": [[0, "shifthappens.tasks.base.T"]], "task (class in shifthappens.tasks.base)": [[0, "shifthappens.tasks.base.Task"]], "taskmetadata (class in shifthappens.task_data.task_metadata)": [[0, "shifthappens.task_data.task_metadata.TaskMetadata"]], "taskregistration (class in shifthappens.task_data.task_registration)": [[0, "shifthappens.task_data.task_registration.TaskRegistration"]], "taskresult (class in shifthappens.tasks.task_result)": [[0, "shifthappens.tasks.task_result.TaskResult"]], "torchdataset (class in shifthappens.data.torch)": [[0, "shifthappens.data.torch.TorchDataset"]], "uncertaintymodelmixin (class in shifthappens.models.mixins)": [[0, "shifthappens.models.mixins.UncertaintyModelMixin"]], "uncertaintytaskmixin (class in shifthappens.tasks.mixins)": [[0, "shifthappens.tasks.mixins.UncertaintyTaskMixin"]], "vgg16 (class in shifthappens.models.torchvision)": [[0, "shifthappens.models.torchvision.VGG16"]], "__torchvisionmodel (class in shifthappens.models.torchvision)": [[0, "shifthappens.models.torchvision.__TorchvisionModel"]], "__torchvisionpreprocessingmixin (class in shifthappens.models.torchvision)": [[0, "shifthappens.models.torchvision.__TorchvisionPreProcessingMixin"]], "_evaluate() (shifthappens.tasks.base.task method)": [[0, "shifthappens.tasks.base.Task._evaluate"]], "_get_imagenet_predictions() (shifthappens.models.base.model method)": [[0, "shifthappens.models.base.Model._get_imagenet_predictions"]], "_predict() (shifthappens.models.base.model method)": [[0, "shifthappens.models.base.Model._predict"]], "_predict() (shifthappens.models.torchvision.__torchvisionmodel method)": [[0, "shifthappens.models.torchvision.__TorchvisionModel._predict"]], "_predict_imagenet_val() (shifthappens.models.base.model method)": [[0, "shifthappens.models.base.Model._predict_imagenet_val"]], "_prepare_dataloader() (shifthappens.tasks.base.task method)": [[0, "shifthappens.tasks.base.Task._prepare_dataloader"]], "abstract_variable() (in module shifthappens.tasks.base)": [[0, "shifthappens.tasks.base.abstract_variable"]], "cache_directory_path (shifthappens.config.config attribute)": [[0, "shifthappens.config.Config.cache_directory_path"]], "cache_predictions() (in module shifthappens.data.imagenet)": [[0, "shifthappens.data.imagenet.cache_predictions"]], "evaluate() (shifthappens.tasks.base.task method)": [[0, "shifthappens.tasks.base.Task.evaluate"]], "evaluate_model() (in module shifthappens.benchmark)": [[0, "shifthappens.benchmark.evaluate_model"]], "get_cached_predictions() (in module shifthappens.data.imagenet)": [[0, "shifthappens.data.imagenet.get_cached_predictions"]], "get_config() (in module shifthappens.config)": [[0, "shifthappens.config.get_config"]], "get_imagenet_validation_loader() (in module shifthappens.data.imagenet)": [[0, "shifthappens.data.imagenet.get_imagenet_validation_loader"]], "get_instance() (shifthappens.config.config class method)": [[0, "shifthappens.config.Config.get_instance"]], "get_registered_tasks() (in module shifthappens.benchmark)": [[0, "shifthappens.benchmark.get_registered_tasks"]], "imagenet21k_preprocessed_validation_path (shifthappens.config.config attribute)": [[0, "shifthappens.config.Config.imagenet21k_preprocessed_validation_path"]], "imagenet_validation_path (shifthappens.config.config attribute)": [[0, "shifthappens.config.Config.imagenet_validation_path"]], "imagenet_validation_result (shifthappens.models.base.model property)": [[0, "shifthappens.models.base.Model.imagenet_validation_result"]], "is_cached() (in module shifthappens.data.imagenet)": [[0, "shifthappens.data.imagenet.is_cached"]], "iterate() (shifthappens.data.base.dataloader method)": [[0, "shifthappens.data.base.DataLoader.iterate"]], "iterate_flavours() (shifthappens.tasks.base.task class method)": [[0, "shifthappens.tasks.base.Task.iterate_flavours"]], "load_imagenet_targets() (in module shifthappens.data.imagenet)": [[0, "shifthappens.data.imagenet.load_imagenet_targets"]], "max_batch_size (shifthappens.data.base.dataloader property)": [[0, "shifthappens.data.base.DataLoader.max_batch_size"]], "module": [[0, "module-shifthappens.benchmark"], [0, "module-shifthappens.config"], [0, "module-shifthappens.data.base"], [0, "module-shifthappens.data.imagenet"], [0, "module-shifthappens.data.torch"], [0, "module-shifthappens.models.base"], [0, "module-shifthappens.models.mixins"], [0, "module-shifthappens.models.torchvision"], [0, "module-shifthappens.task_data.task_metadata"], [0, "module-shifthappens.task_data.task_registration"], [0, "module-shifthappens.tasks.base"], [0, "module-shifthappens.tasks.metrics"], [0, "module-shifthappens.tasks.mixins"], [0, "module-shifthappens.tasks.task_result"]], "parameter() (in module shifthappens.tasks.base)": [[0, "shifthappens.tasks.base.parameter"]], "predict() (shifthappens.models.base.model method)": [[0, "shifthappens.models.base.Model.predict"]], "prepare() (shifthappens.models.base.model method)": [[0, "shifthappens.models.base.Model.prepare"]], "register_task() (in module shifthappens.benchmark)": [[0, "shifthappens.benchmark.register_task"]], "setup() (shifthappens.tasks.base.task method)": [[0, "shifthappens.tasks.base.Task.setup"]], "shifthappens.benchmark": [[0, "module-shifthappens.benchmark"]], "shifthappens.config": [[0, "module-shifthappens.config"]], "shifthappens.data.base": [[0, "module-shifthappens.data.base"]], "shifthappens.data.imagenet": [[0, "module-shifthappens.data.imagenet"]], "shifthappens.data.torch": [[0, "module-shifthappens.data.torch"]], "shifthappens.models.base": [[0, "module-shifthappens.models.base"]], "shifthappens.models.mixins": [[0, "module-shifthappens.models.mixins"]], "shifthappens.models.torchvision": [[0, "module-shifthappens.models.torchvision"]], "shifthappens.task_data.task_metadata": [[0, "module-shifthappens.task_data.task_metadata"]], "shifthappens.task_data.task_registration": [[0, "module-shifthappens.task_data.task_registration"]], "shifthappens.tasks.base": [[0, "module-shifthappens.tasks.base"]], "shifthappens.tasks.metrics": [[0, "module-shifthappens.tasks.metrics"]], "shifthappens.tasks.mixins": [[0, "module-shifthappens.tasks.mixins"]], "shifthappens.tasks.task_result": [[0, "module-shifthappens.tasks.task_result"]], "shuffle_data() (in module shifthappens.data.base)": [[0, "shifthappens.data.base.shuffle_data"]], "variable() (in module shifthappens.tasks.base)": [[0, "shifthappens.tasks.base.variable"]], "verbose (shifthappens.config.config attribute)": [[0, "shifthappens.config.Config.verbose"]]}})