refactor instance name; re-run to get error outputs

harsha-simhadri · Aug 29, 2024 · d3f652d · d3f652d
1 parent d1d5094
commit d3f652d
Show file tree

Hide file tree

Showing 60 changed files with 2,274 additions and 0 deletions.
diff --git a/neurips23/latitude/_latitude-m4-bare-medium.md b/neurips23/latitude/_latitude-m4-bare-medium.md
diff --git a/neurips23/latitude/commands/filter__cufe.sh b/neurips23/latitude/commands/filter__cufe.sh
diff --git a/neurips23/latitude/commands/filter__dhq.sh b/neurips23/latitude/commands/filter__dhq.sh
diff --git a/neurips23/latitude/commands/filter__faiss.sh b/neurips23/latitude/commands/filter__faiss.sh
diff --git a/neurips23/latitude/commands/filter__faissplus.sh b/neurips23/latitude/commands/filter__faissplus.sh
diff --git a/neurips23/latitude/commands/filter__fdufilterdiskann.sh b/neurips23/latitude/commands/filter__fdufilterdiskann.sh
diff --git a/neurips23/latitude/commands/filter__hwtl_sdu_anns_filter.sh b/neurips23/latitude/commands/filter__hwtl_sdu_anns_filter.sh
diff --git a/neurips23/latitude/commands/filter__parlayivf.sh b/neurips23/latitude/commands/filter__parlayivf.sh
diff --git a/neurips23/latitude/commands/filter__pinecone.sh b/neurips23/latitude/commands/filter__pinecone.sh
diff --git a/neurips23/latitude/commands/filter__puck.sh b/neurips23/latitude/commands/filter__puck.sh
diff --git a/neurips23/latitude/commands/filter__pyanns.sh b/neurips23/latitude/commands/filter__pyanns.sh
diff --git a/neurips23/latitude/commands/filter__wm_filter.sh b/neurips23/latitude/commands/filter__wm_filter.sh
diff --git a/neurips23/latitude/commands/filter__zilliz.sh b/neurips23/latitude/commands/filter__zilliz.sh
diff --git a/neurips23/latitude/commands/ood__cufe.sh b/neurips23/latitude/commands/ood__cufe.sh
diff --git a/neurips23/latitude/commands/ood__diskann.sh b/neurips23/latitude/commands/ood__diskann.sh
diff --git a/neurips23/latitude/commands/ood__epsearch.sh b/neurips23/latitude/commands/ood__epsearch.sh
diff --git a/neurips23/latitude/commands/ood__mysteryann-dif.sh b/neurips23/latitude/commands/ood__mysteryann-dif.sh
diff --git a/neurips23/latitude/commands/ood__mysteryann.sh b/neurips23/latitude/commands/ood__mysteryann.sh
diff --git a/neurips23/latitude/commands/ood__ngt.sh b/neurips23/latitude/commands/ood__ngt.sh
diff --git a/neurips23/latitude/commands/ood__pinecone-ood.sh b/neurips23/latitude/commands/ood__pinecone-ood.sh
diff --git a/neurips23/latitude/commands/ood__puck-fizz.sh b/neurips23/latitude/commands/ood__puck-fizz.sh
diff --git a/neurips23/latitude/commands/ood__puck.sh b/neurips23/latitude/commands/ood__puck.sh
diff --git a/neurips23/latitude/commands/ood__pyanns.sh b/neurips23/latitude/commands/ood__pyanns.sh
diff --git a/neurips23/latitude/commands/ood__scann.sh b/neurips23/latitude/commands/ood__scann.sh
diff --git a/neurips23/latitude/commands/ood__sustech-ood.sh b/neurips23/latitude/commands/ood__sustech-ood.sh
diff --git a/neurips23/latitude/commands/ood__vamana.sh b/neurips23/latitude/commands/ood__vamana.sh
diff --git a/neurips23/latitude/commands/ood__zilliz.sh b/neurips23/latitude/commands/ood__zilliz.sh
diff --git a/neurips23/latitude/commands/sparse__cufe.sh b/neurips23/latitude/commands/sparse__cufe.sh
diff --git a/neurips23/latitude/commands/sparse__linscan.sh b/neurips23/latitude/commands/sparse__linscan.sh
diff --git a/neurips23/latitude/commands/sparse__nle.sh b/neurips23/latitude/commands/sparse__nle.sh
diff --git a/neurips23/latitude/commands/sparse__pinecone_smips.sh b/neurips23/latitude/commands/sparse__pinecone_smips.sh
diff --git a/neurips23/latitude/commands/sparse__pyanns.sh b/neurips23/latitude/commands/sparse__pyanns.sh
diff --git a/neurips23/latitude/commands/sparse__shnsw.sh b/neurips23/latitude/commands/sparse__shnsw.sh
diff --git a/neurips23/latitude/commands/sparse__spmat.sh b/neurips23/latitude/commands/sparse__spmat.sh
diff --git a/neurips23/latitude/commands/sparse__sustech-whu.sh b/neurips23/latitude/commands/sparse__sustech-whu.sh
diff --git a/neurips23/latitude/commands/sparse__zilliz.sh b/neurips23/latitude/commands/sparse__zilliz.sh
diff --git a/neurips23/latitude/commands/streaming__cufe.sh b/neurips23/latitude/commands/streaming__cufe.sh
diff --git a/neurips23/latitude/commands/streaming__diskann.sh b/neurips23/latitude/commands/streaming__diskann.sh
diff --git a/neurips23/latitude/commands/streaming__hwtl_sdu_anns_stream.sh b/neurips23/latitude/commands/streaming__hwtl_sdu_anns_stream.sh
diff --git a/neurips23/latitude/commands/streaming__pinecone.sh b/neurips23/latitude/commands/streaming__pinecone.sh
diff --git a/neurips23/latitude/commands/streaming__puck.sh b/neurips23/latitude/commands/streaming__puck.sh
diff --git a/neurips23/latitude/commands/streaming__pyanns.sh b/neurips23/latitude/commands/streaming__pyanns.sh
diff --git a/neurips23/latitude/commands/streaming__scann.sh b/neurips23/latitude/commands/streaming__scann.sh
diff --git a/neurips23/latitude/data_export_m2-medium.csv b/neurips23/latitude/data_export_m2-medium.csv
@@ -246,3 +246,33 @@ wm_filter,"Faiss(('IVF1024,SQ8', 'direct', {'nprobe': 160, 'max_codes': 1800, 's
 wm_filter,"Faiss(('IVF1024,SQ8', 'direct', {'nprobe': 130, 'max_codes': 2100, 'selector_probe_limit': 130}))",yfcc-10M,10,19596.002144658287,0.0,314.16982865333557,6424608.0,0,0,filter,0.915053
 wm_filter,"Faiss(('IVF1024,SQ8', 'direct', {'nprobe': 160, 'max_codes': 2100, 'selector_probe_limit': 160}))",yfcc-10M,10,19028.020790121896,0.0,314.16982865333557,6424608.0,0,0,filter,0.91951
 wm_filter,"Faiss(('IVF1024,SQ8', 'direct', {'nprobe': 200, 'max_codes': 2100, 'selector_probe_limit': 200}))",yfcc-10M,10,18333.36363903393,0.0,314.16982865333557,6424608.0,0,0,filter,0.9222340000000001
+zilliz,zilliz_qdrop0.12_ef45,sparse-full,10,37323.7214793208,0.0,5274.822802066803,29272756.0,0,0,sparse,0.895272206303725
+zilliz,zilliz_qdrop0.11_ef45,sparse-full,10,36991.01376980246,0.0,5274.822802066803,29272756.0,0,0,sparse,0.8958022922636102
+zilliz,zilliz_qdrop0.12_ef50,sparse-full,10,34932.80647389905,0.0,5274.822802066803,29272756.0,0,0,sparse,0.9023925501432665
+zilliz,zilliz_qdrop0.12_ef55,sparse-full,10,32481.482091286012,0.0,5274.822802066803,29272756.0,0,0,sparse,0.9078510028653295
+zilliz,zilliz_qdrop0.11_ef55,sparse-full,10,32208.149795922858,0.0,5274.822802066803,29272756.0,0,0,sparse,0.909598853868195
+zilliz,zilliz_qdrop0.12_ef60,sparse-full,10,30775.01597289601,0.0,5274.822802066803,29272756.0,0,0,sparse,0.912650429799427
+zilliz,zilliz_qdrop0.12_ef65,sparse-full,10,29048.271088497473,0.0,5274.822802066803,29272756.0,0,0,sparse,0.9178080229226362
+zilliz,zilliz_qdrop0.11_ef65,sparse-full,10,28489.920124562086,0.0,5274.822802066803,29272756.0,0,0,sparse,0.9210315186246418
+zilliz,zilliz_qdrop0.12_ef70,sparse-full,10,27393.846372656535,0.0,5274.822802066803,29272756.0,0,0,sparse,0.9215472779369627
+zilliz,zilliz_qdrop0.11_ef70,sparse-full,10,27185.03323815493,0.0,5274.822802066803,29272756.0,0,0,sparse,0.9248280802292264
+zilliz,zilliz_(R48_L500)_ef90,text2image-10M,10,81895.28745720285,0.0,4123.189875841141,11282784.0,0,0,ood,0.8895
+zilliz,zilliz_(R48_L500)_ef95,text2image-10M,10,78612.61969094022,0.0,4123.189875841141,11282784.0,0,0,ood,0.8942209999999999
+zilliz,zilliz_(R48_L500)_ef100,text2image-10M,10,75614.953440862,0.0,4123.189875841141,11282784.0,0,0,ood,0.898085
+zilliz,zilliz_(R48_L500)_ef102,text2image-10M,10,74507.65273494678,0.0,4123.189875841141,11282784.0,0,0,ood,0.8999159999999999
+zilliz,zilliz_(R48_L500)_ef104,text2image-10M,10,73337.07044359592,0.0,4123.189875841141,11282784.0,0,0,ood,0.901492
+zilliz,zilliz_(R48_L500)_ef106,text2image-10M,10,72220.21816363529,0.0,4123.189875841141,11282784.0,0,0,ood,0.9029870000000001
+zilliz,zilliz_(R48_L500)_ef108,text2image-10M,10,71334.43031408514,0.0,4123.189875841141,11282784.0,0,0,ood,0.904517
+zilliz,zilliz_(R48_L500)_ef110,text2image-10M,10,70211.1051722617,0.0,4123.189875841141,11282784.0,0,0,ood,0.905845
+zilliz,zilliz_(R48_L500)_ef115,text2image-10M,10,67960.34734715418,0.0,4123.189875841141,11282784.0,0,0,ood,0.9092100000000001
+zilliz,zilliz_(R48_L500)_ef120,text2image-10M,10,65656.59269486376,0.0,4123.189875841141,11282784.0,0,0,ood,0.912359
+zilliz,filter_searcher_R12_L100_ef16,yfcc-10M,10,224055.65835751436,0.0,-1.0,12818012.0,0,0,filter,0.892301
+zilliz,filter_searcher_R12_L100_ef18,yfcc-10M,10,220566.86940800442,0.0,-1.0,12818012.0,0,0,filter,0.8992739999999999
+zilliz,filter_searcher_R12_L100_ef19,yfcc-10M,10,220846.29495849824,0.0,-1.0,12818012.0,0,0,filter,0.9025650000000001
+zilliz,filter_searcher_R12_L100_ef20,yfcc-10M,10,220576.14900064265,0.0,-1.0,12818012.0,0,0,filter,0.9054249999999999
+zilliz,filter_searcher_R12_L100_ef22,yfcc-10M,10,219635.82056100998,0.0,-1.0,12818012.0,0,0,filter,0.910566
+zilliz,filter_searcher_R12_L100_ef24,yfcc-10M,10,217391.39899418413,0.0,-1.0,12818012.0,0,0,filter,0.915068
+zilliz,filter_searcher_R12_L100_ef26,yfcc-10M,10,216789.8102735623,0.0,-1.0,12818012.0,0,0,filter,0.918776
+zilliz,filter_searcher_R12_L100_ef28,yfcc-10M,10,215102.2199976204,0.0,-1.0,12818012.0,0,0,filter,0.9220560000000001
+zilliz,filter_searcher_R12_L100_ef32,yfcc-10M,10,212919.74867747026,0.0,-1.0,12818012.0,0,0,filter,0.927543
+zilliz,filter_searcher_R12_L100_ef36,yfcc-10M,10,210595.59488441428,0.0,-1.0,12818012.0,0,0,filter,0.932306
diff --git a/neurips23/latitude/data_export_m4-bare-medium.csv b/neurips23/latitude/data_export_m4-bare-medium.csv
diff --git a/neurips23/latitude/errors/filter__dhq.txt b/neurips23/latitude/errors/filter__dhq.txt
@@ -0,0 +1,217 @@
++ set -e
++ python install.py --neurips23track filter --algorithm dhq
+#0 building with "default" instance using docker driver
+
+#1 [internal] load build definition from Dockerfile
+#1 transferring dockerfile: 556B done
+#1 DONE 0.0s
+
+#2 [internal] load metadata for docker.io/library/ubuntu:jammy
+#2 DONE 0.1s
+
+#3 [internal] load .dockerignore
+#3 transferring context: 66B done
+#3 DONE 0.0s
+
+#4 [1/8] FROM docker.io/library/ubuntu:jammy@sha256:adbb90115a21969d2fe6fa7f9af4253e16d45f8d4c1e930182610c4731962658
+#4 DONE 0.0s
+
+#5 [internal] load build context
+#5 transferring context: 321B done
+#5 DONE 0.0s
+
+#6 [3/8] RUN wget https://aka.ms/downloadazcopy-v10-linux && mv downloadazcopy-v10-linux azcopy.tgz && tar xzf azcopy.tgz --transform 's!^[^/]\+\($\|/\)!azcopy_folder\1!'
+#6 CACHED
+
+#7 [5/8] RUN pip3 install -U pip
+#7 CACHED
+
+#8 [7/8] COPY requirements_py3.10.txt run_algorithm.py ./
+#8 CACHED
+
+#9 [2/8] RUN apt-get update && apt-get install -y python3-numpy python3-scipy python3-pip build-essential git axel wget
+#9 CACHED
+
+#10 [4/8] RUN cp azcopy_folder/azcopy /usr/bin
+#10 CACHED
+
+#11 [6/8] WORKDIR /home/app
+#11 CACHED
+
+#12 [8/8] RUN pip3 install -r requirements_py3.10.txt
+#12 CACHED
+
+#13 exporting to image
+#13 exporting layers done
+#13 writing image sha256:2c76bbc113266e4782270104c7c70525aab0dcc93687ee5293516e33e09661ca done
+#13 naming to docker.io/library/neurips23 done
+#13 DONE 0.0s
+#0 building with "default" instance using docker driver
+
+#1 [internal] load build definition from Dockerfile
+#1 transferring dockerfile: 1.76kB done
+#1 DONE 0.0s
+
+#2 [internal] load metadata for docker.io/library/neurips23:latest
+#2 DONE 0.0s
+
+#3 [internal] load .dockerignore
+#3 transferring context: 66B done
+#3 DONE 0.0s
+
+#4 [ 1/22] FROM docker.io/library/neurips23:latest
+#4 DONE 0.0s
+
+#5 [internal] load build context
+#5 transferring context: 185B done
+#5 DONE 0.0s
+
+#6 [ 2/22] RUN apt update
+#6 CACHED
+
+#7 [14/22] RUN cp /home/app/DHQ/faiss/build/faiss/libfaiss.so /lib
+#7 CACHED
+
+#8 [ 6/22] RUN DEBIAN_FRONTEND=noninteractive apt install -y git make cmake g++ libaio-dev libgoogle-perftools-dev libunwind-dev clang-format libboost-dev libboost-program-options-dev libmkl-full-dev libcpprest-dev python3.10 python3-numpy python3-scipy python3-pip build-essential git libblas-dev liblapack-dev wget libaio-dev libgoogle-perftools-dev clang-format libboost-all-dev libopenblas-dev liblapacke-dev
+#8 CACHED
+
+#9 [ 7/22] RUN apt-get update; DEBIAN_FRONTEND=noninteractive apt install intel-mkl python3-setuptools  wget python3-matplotlib build-essential checkinstall libssl-dev swig4.0 python3-dev  python3-numpy  python3-numpy-dev   -y
+#9 CACHED
+
+#10 [17/22] RUN cp /home/app/DHQ/faiss/build/faiss/python/_swigfaiss.so /lib
+#10 CACHED
+
+#11 [18/22] RUN cd /home/app/DHQ/faiss/build/faiss/python/ && python3 setup.py install
+#11 CACHED
+
+#12 [ 9/22] RUN python3 -m pip install -r requirements_conda.txt
+#12 CACHED
+
+#13 [13/22] RUN cp /home/app/DHQ/faiss/build/faiss/python/libfaiss_python_callbacks.so /lib
+#13 CACHED
+
+#14 [ 5/22] RUN apt update
+#14 CACHED
+
+#15 [10/22] RUN pip3 install -U pip numpy pybind11 tqdm
+#15 CACHED
+
+#16 [16/22] RUN cp /home/app/DHQ/faiss/build/faiss/python/_swigfaiss_avx2.so /lib
+#16 CACHED
+
+#17 [15/22] RUN cp /home/app/DHQ/faiss/build/faiss/libfaiss_avx2.so /lib
+#17 CACHED
+
+#18 [ 3/22] RUN apt install -y software-properties-common
+#18 CACHED
+
+#19 [11/22] RUN git clone https://github.com/SDU-L/DHQ.git
+#19 CACHED
+
+#20 [12/22] RUN chmod -R +777 DHQ/
+#20 CACHED
+
+#21 [ 4/22] RUN add-apt-repository -y ppa:git-core/ppa
+#21 CACHED
+
+#22 [ 8/22] COPY install/requirements_conda.txt ./
+#22 CACHED
+
+#23 [19/22] RUN cd /home/app
+#23 CACHED
+
+#24 [20/22] RUN python3 -c 'import faiss; print(faiss.IDSelectorFilterWise); print(faiss.__version__)'
+#24 0.325 
+#24 0.325 A module that was compiled using NumPy 1.x cannot be run in
+#24 0.325 NumPy 2.1.0 as it may crash. To support both 1.x and 2.x
+#24 0.325 versions of NumPy, modules must be compiled with NumPy 2.0.
+#24 0.325 Some module may need to rebuild instead e.g. with 'pybind11>=2.12'.
+#24 0.325 
+#24 0.325 If you are a user of the module, the easiest solution will be to
+#24 0.325 downgrade to 'numpy<2' or try to upgrade the affected module.
+#24 0.325 We expect that some modules will need time to support NumPy 2.
+#24 0.325 
+#24 0.325 Traceback (most recent call last):  File "<string>", line 1, in <module>
+#24 0.325   File "/home/app/DHQ/faiss/build/faiss/python/build/lib/faiss/__init__.py", line 16, in <module>
+#24 0.325     from .loader import *
+#24 0.325   File "/home/app/DHQ/faiss/build/faiss/python/build/lib/faiss/loader.py", line 55, in <module>
+#24 0.325     from .swigfaiss_avx2 import *
+#24 0.325   File "/home/app/DHQ/faiss/build/faiss/python/build/lib/faiss/swigfaiss_avx2.py", line 13, in <module>
+#24 0.325     from . import _swigfaiss_avx2
+#24 0.325 AttributeError: _ARRAY_API not found
+#24 0.387 
+#24 0.387 A module that was compiled using NumPy 1.x cannot be run in
+#24 0.387 NumPy 2.1.0 as it may crash. To support both 1.x and 2.x
+#24 0.387 versions of NumPy, modules must be compiled with NumPy 2.0.
+#24 0.387 Some module may need to rebuild instead e.g. with 'pybind11>=2.12'.
+#24 0.387 
+#24 0.387 If you are a user of the module, the easiest solution will be to
+#24 0.387 downgrade to 'numpy<2' or try to upgrade the affected module.
+#24 0.387 We expect that some modules will need time to support NumPy 2.
+#24 0.387 
+#24 0.387 Traceback (most recent call last):  File "<string>", line 1, in <module>
+#24 0.387   File "/home/app/DHQ/faiss/build/faiss/python/build/lib/faiss/__init__.py", line 16, in <module>
+#24 0.387     from .loader import *
+#24 0.387   File "/home/app/DHQ/faiss/build/faiss/python/build/lib/faiss/loader.py", line 65, in <module>
+#24 0.387     from .swigfaiss import *
+#24 0.387   File "/home/app/DHQ/faiss/build/faiss/python/build/lib/faiss/swigfaiss.py", line 13, in <module>
+#24 0.387     from . import _swigfaiss
+#24 0.387 AttributeError: _ARRAY_API not found
+#24 0.387 Traceback (most recent call last):
+#24 0.387   File "<string>", line 1, in <module>
+#24 0.387   File "/home/app/DHQ/faiss/build/faiss/python/build/lib/faiss/__init__.py", line 16, in <module>
+#24 0.387     from .loader import *
+#24 0.387   File "/home/app/DHQ/faiss/build/faiss/python/build/lib/faiss/loader.py", line 65, in <module>
+#24 0.387     from .swigfaiss import *
+#24 0.387   File "/home/app/DHQ/faiss/build/faiss/python/build/lib/faiss/swigfaiss.py", line 13, in <module>
+#24 0.387     from . import _swigfaiss
+#24 0.387 ImportError: numpy.core.multiarray failed to import
+#24 ERROR: process "/bin/sh -c python3 -c 'import faiss; print(faiss.IDSelectorFilterWise); print(faiss.__version__)'" did not complete successfully: exit code: 1
+------
+ > [20/22] RUN python3 -c 'import faiss; print(faiss.IDSelectorFilterWise); print(faiss.__version__)':
+0.387 AttributeError: _ARRAY_API not found
+0.387 Traceback (most recent call last):
+0.387   File "<string>", line 1, in <module>
+0.387   File "/home/app/DHQ/faiss/build/faiss/python/build/lib/faiss/__init__.py", line 16, in <module>
+0.387     from .loader import *
+0.387   File "/home/app/DHQ/faiss/build/faiss/python/build/lib/faiss/loader.py", line 65, in <module>
+0.387     from .swigfaiss import *
+0.387   File "/home/app/DHQ/faiss/build/faiss/python/build/lib/faiss/swigfaiss.py", line 13, in <module>
+0.387     from . import _swigfaiss
+0.387 ImportError: numpy.core.multiarray failed to import
+------
+Dockerfile:30
+--------------------
+  28 |     
+  29 |     ENV PYTHONPATH=DHQ/faiss/build/faiss/python/build/lib/
+  30 | >>> RUN python3 -c 'import faiss; print(faiss.IDSelectorFilterWise); print(faiss.__version__)'
+  31 |     RUN pip3 install DHQ/DHQ-1.0.3-cp310-cp310-linux_x86_64.whl
+  32 |     RUN python3 -c 'import DHQ'
+--------------------
+ERROR: failed to solve: process "/bin/sh -c python3 -c 'import faiss; print(faiss.IDSelectorFilterWise); print(faiss.__version__)'" did not complete successfully: exit code: 1
+Building base image...
+Building algorithm images... with (1) processes
+Building neurips23-filter-dhq...
+docker build  --rm -t neurips23-filter-dhq -f neurips23/filter/dhq/Dockerfile .
+
+
+Install Status:
+{'neurips23-filter-dhq': 'fail'}
++ python3 run.py --dataset yfcc-10M --algorithm dhq --neurips23track filter
+file data/yfcc100M/query.public.100K.u8bin already exists
+file data/yfcc100M/GT.public.ibin already exists
+file data/yfcc100M/query.private.2727415019.100K.u8bin already exists
+file data/yfcc100M/GT.private.2727415019.ibin already exists
+file data/yfcc100M/base.10M.u8bin already exists
+file data/yfcc100M/base.metadata.10M.spmat already exists
+file data/yfcc100M/query.metadata.public.100K.spmat already exists
+file data/yfcc100M/query.metadata.private.2727415019.100K.spmat already exists
+2024-08-29 20:12:06,728 - annb - INFO - running only dhq
+2024-08-29 20:12:06,820 - annb - INFO - not all docker images available, only: {'neurips23-streaming-pyanns', 'neurips23-filter-faiss', 'neurips23-sparse-cufe', 'neurips23-streaming-pinecone', 'neurips23-streaming-cufe', 'neurips23-ood-mysteryann-dif', 'neurips23-ood-vamana', 'neurips23-ood-ngt', 'neurips23-streaming-hwtl_sdu_anns_stream', 'neurips23-ood-pyanns', 'neurips23-sparse-shnsw', 'neurips23-streaming-puck', 'neurips23-ood-mysteryann', 'neurips23-ood-pinecone-ood', 'neurips23-ood-sustech-ood', 'neurips23-ood-cufe', 'neurips23-filter-wm_filter', 'neurips23-filter-faissplus', 'neurips23-sparse-spmat', 'neurips23-streaming-diskann', 'neurips23-sparse-linscan', 'neurips23-filter-puck', 'neurips23-streaming-scann', 'neurips23-filter-parlayivf', 'neurips23-filter-cufe', 'neurips23-ood-zilliz', 'neurips23-sparse-zilliz', 'neurips23-ood-puck', 'neurips23-sparse-nle', 'neurips23', 'neurips23-filter-pinecone', 'neurips23-ood-epsearch', 'neurips23-filter-pyanns', 'neurips23-filter-zilliz', 'neurips23-ood-diskann', 'neurips23-sparse-pyanns', 'neurips23-sparse-pinecone_smips'}
+2024-08-29 20:12:06,820 - annb - INFO - missing docker images: {'neurips23-filter-dhq'}
+Traceback (most recent call last):
+  File "/home/gwilliams/Projects/BigANN/big-ann-benchmarks/run.py", line 6, in <module>
+    main()
+  File "/home/gwilliams/Projects/BigANN/big-ann-benchmarks/benchmark/main.py", line 257, in main
+    raise Exception('Nothing to run')
+Exception: Nothing to run