From da285066d2f7ce758371bb4be5279b04db337338 Mon Sep 17 00:00:00 2001 From: George Williams Date: Thu, 29 Aug 2024 16:03:22 -0700 Subject: [PATCH] cleanup and re-render markdown --- neurips23/latitude-m4-metal-medium.md | 610 +++-------- .../latitude/_latitude-m4-metal-medium.md | 332 +----- neurips23/latitude/analysis.ipynb | 996 +++++++++--------- 3 files changed, 672 insertions(+), 1266 deletions(-) diff --git a/neurips23/latitude-m4-metal-medium.md b/neurips23/latitude-m4-metal-medium.md index c7ac6665..fb3c367e 100644 --- a/neurips23/latitude-m4-metal-medium.md +++ b/neurips23/latitude-m4-metal-medium.md @@ -13,7 +13,7 @@ The NeurIPS2023 Practical Vector Search Challenge evaluated participating algorithms on Azure and EC2 CPU-based hardware instances. -In pursuit of expanding the evaluation criteria, we are evaluating on other generally available hardware configurations. +In pursuit of expanding the evaluation criteria, we are also running on other generally available hardware configurations. Shown here are results run on the following hardware: * AMD EPYC 9124 16-Core 3GHz processor @@ -26,184 +26,189 @@ Note: * The [NeurIPS2023](README.md) Docker-based eval limits the use of available underlying resources. ## Results - +
- - - + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + +
filtersparseoodfiltersparseood
rankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqps
1
zilliz
 213.3K1
zilliz
 34.8K1
pinecone-ood
 76.9K1
zilliz
 213.3K1
zilliz
 34.8K1
pinecone-ood
 76.9K
2
pinecone
 146.7K2
pyanns
 26.9K2
zilliz
 73.5K2
pinecone
 146.7K2
pyanns
 26.9K2
zilliz
 73.5K
3
puck
 62.3K3
pinecone_smips
 12.0K3
pyanns
 55.5K3
puck
 62.3K3
pinecone_smips
 12.0K3
pyanns
 55.5K
4
parlayivf
 55.0K4
shnsw
 8.2K4
scann
 32.3K4
parlayivf
 55.0K4
shnsw
 8.2K4
scann
 32.3K
5
wm_filter
 20.9K5
nle
 2.9K5
sustech-ood
 28.5K5
wm_filter
 20.9K5
nle
 2.9K5
sustech-ood
 28.5K
6
pyanns
 9.0K6
cufe
 0.1K6
mysteryann-dif
 27.9K6
pyanns
 9.0K6
cufe
 0.1K6
mysteryann-dif
 27.9K
7
faissplus
 8.5K7
linscan
 0.1K7
mysteryann
 26.6K7
faissplus
 8.5K7
linscan
 0.1K7
mysteryann
 26.6K
8
faiss
 7.3K
sustech-whu
8
vamana
 20.0K8
faiss
 7.3K
sustech-whu
8
vamana
 20.0K
9
cufe
 6.3K
spmat
9
puck
 19.0K9
cufe
 6.3K
spmat
9
puck
 19.0K
hwtl_sdu_anns_filter
10
ngt
 11.9K
fdufilterdiskann
10
ngt
 11.9K
fdufilterdiskann
11
epsearch
 7.7K
hwtl_sdu_anns_filter
11
epsearch
 7.7K
dhq
12
diskann
 6.4K
dhq
12
diskann
 6.4K
13
cufe
 5.4K13
cufe
 5.4K
puck-fizz
puck-fizz
+Notes: +* Evaluations were run in late August 2024 +* Click on algorithm link to see the build and run commmand used (and errors if any). +* Pareto graphs for each track shown below. + ### Track: Filter ![Filter](latitude/filter.png) @@ -218,7 +223,7 @@ Note: ### Track: Streaming -![Streaming](latitude/streaming.png) +TODO ### Data Export @@ -259,160 +264,12 @@ Prepare the track dataset by running the following command in the top-level dire python create_dataset.py --dataset sparse-full ``` -#### Sparse Algorithm: cufe - -``` -python install.py --neurips23track sparse --algorithm cufe -python -m pip install requests==2.31.0 # urllib3.exceptions.URLSchemeUnknown: Not supported URL scheme http+docker -``` - -#### Sparse Algorithm: linscan - -``` -python install.py --neurips23track sparse --algorithm linscan -python run.py --dataset sparse-full --algorithm linscan --neurips23track sparse -``` - -#### Sparse Algorithm: nle - -``` -python install.py --neurips23track sparse --algorithm nle -python run.py --dataset sparse-full --algorithm nle --neurips23track sparse -``` - -#### Sparse Algorithm: pyanns - -``` -python install.py --neurips23track sparse --algorithm pyanns -python run.py --dataset sparse-full --algorithm pyanns --neurips23track sparse -``` - -#### Sparse Algorithm: shnsw - -``` -python install.py --neurips23track sparse --algorithm shnsw -python run.py --dataset sparse-full --algorithm shnsw --neurips23track sparse -``` - -#### Sparse Algorithm: spmat - -``` -python install.py --neurips23track sparse --algorithm spmat -python run.py --dataset sparse-full --algorithm spmat --neurips23track sparse # ERROR: sparse-full is not in spmat/config.yaml -``` - -#### Sparse Algorithm: sustech-whu - -``` -python install.py --neurips23track sparse --algorithm sustech-whu -# ERROR: git could not clone 'https://github.com/lizzy-0323/SUSTech-WHU-Sparse.git' -``` - -### Sparse Algorithm: pinecone_smips - -``` -python install.py --neurips23track sparse --algorithm pinecone_smips -python run.py --dataset sparse-full --algorithm pinecone_smips --neurips23track sparse -``` - -#### Sparse Algorithm: zilliz - -``` -python install.py --neurips23track sparse --algorithm zilliz -python run.py --dataset sparse-full --algorithm zilliz --neurips23track sparse -``` - ### Filter Track Prepare the track dataset by running the following command in the top-level directory of the repository: ``` python create_dataset.py --dataset yfcc-10M ``` - -#### Filter Algorithm: cufe - -``` -python install.py --neurips23track filter --algorithm cufe -python3 run.py --dataset yfcc-10M --algorithm cufe --neurips23track filter -``` - -#### Filter Algorithm: dhq - -``` -python install.py --neurips23track filter --algorithm dhq -# ERROR: failed to solve: process "/bin/sh -c python3 -c 'import faiss; print(faiss.IDSelectorFilterWise); print(faiss.__version__)'" did not complete successfully: exit code: 1 -``` - -#### Filter Algorithm: faiss - -``` -python install.py --neurips23track filter --algorithm faiss -python3 run.py --dataset yfcc-10M --algorithm faiss --neurips23track filter -``` - -#### Filter Algorithm: faissplus - -``` -python install.py --neurips23track filter --algorithm faissplus -python3 run.py --dataset yfcc-10M --algorithm faissplus --neurips23track filter -``` - -#### Filter Algorithm: fdufilterdiskann - -``` -python install.py --neurips23track filter --algorithm fdufilterdiskann -# ERROR: failed to solve: process "/bin/sh -c git clone --recursive --branch main https://github.com/PUITAR/FduFilterDiskANN.git" did not complete successfully: exit code: 128 -``` - -#### Filter Algorithm: hwtl_sdu_anns_filter - -``` -python install.py --neurips23track filter --algorithm hwtl_sdu_anns_filter -# ERROR: failed to solve: process "/bin/sh -c python3 -c 'import faiss; print(faiss.IndexFlatL2); print(faiss.__version__)'" did not complete successfully: exit code: 1 -``` - -#### Filter Algorithm: parlayivf - -``` -python install.py --neurips23track filter --algorithm parlayivf -python3 run.py --dataset yfcc-10M --algorithm parlayivf --neurips23track filter -``` - -#### Filter Algorithm: puck - -``` -python install.py --neurips23track filter --algorithm puck -python3 run.py --dataset yfcc-10M --algorithm puck --neurips23track filter -``` - -#### Filter Algorithm: pyanns - -``` -python install.py --neurips23track filter --algorithm pyanns -python3 run.py --dataset yfcc-10M --algorithm pyanns --neurips23track filter -``` - -#### Filter Algorithm: wm_filter - -``` -python install.py --neurips23track filter --algorithm wm_filter -python3 run.py --dataset yfcc-10M --algorithm wm_filter --neurips23track filter -``` - -#### Filter Algorithm: pinecone - -``` -python install.py --neurips23track filter --algorithm pinecone -python run.py --neurips23track filter --algorithm pinecone --dataset yfcc-10M -``` - -#### Filter Algorithm: zilliz - -``` -python install.py --neurips23track filter --algorithm zilliz -python3 run.py --dataset yfcc-10M --algorithm zilliz --neurips23track filter -``` - ### OOD Track Prepare the track dataset by running the following command in the top-level directory of the repository: @@ -420,97 +277,6 @@ Prepare the track dataset by running the following command in the top-level dire python create_dataset.py --dataset text2image-10M ``` -#### OOD Algorithm: cufe - -``` -python install.py --neurips23track ood --algorithm cufe -python3 run.py --dataset text2image-10M --algorithm cufe --neurips23track ood -``` - -#### OOD Algorithm: diskann - -``` -python install.py --neurips23track ood --algorithm diskann -python3 run.py --dataset text2image-10M --algorithm diskann --neurips23track ood -``` - -#### OOD Algorithm: epsearch - -``` -python install.py --neurips23track ood --algorithm epsearch -python3 run.py --dataset text2image-10M --algorithm epsearch --neurips23track ood -``` - -#### OOD Algorithm: mysteryann - -``` -python install.py --neurips23track ood --algorithm mysteryann -python3 run.py --dataset text2image-10M --algorithm mysteryann --neurips23track ood -``` - -#### OOD Algorithm: mysteryann-dif - -``` -python install.py --neurips23track ood --algorithm mysteryann-dif -python3 run.py --dataset text2image-10M --algorithm mysteryann-dif --neurips23track ood -``` - -#### OOD Algorithm: ngt - -``` -python install.py --neurips23track ood --algorithm ngt -python3 run.py --dataset text2image-10M --algorithm ngt --neurips23track ood -``` - -#### OOD Algorithm: puck - -``` -python install.py --neurips23track ood --algorithm puck -python3 run.py --dataset text2image-10M --algorithm puck --neurips23track ood -``` - -#### OOD Algorithm: puck-fizz - -``` -python install.py --neurips23track ood --algorithm puck-fizz -ERROR: failed to solve: process "/bin/sh -c git clone -b ood-try https://github.com/baidu/puck.git" did not complete successfully: exit code: 128 -``` - -#### OOD Algorithm: pyanns - -``` -python install.py --neurips23track ood --algorithm pyanns -python3 run.py --dataset text2image-10M --algorithm pyanns --neurips23track ood -``` - -#### OOD Algorithm: sustech-ood - -``` -python install.py --neurips23track ood --algorithm sustech-ood -python3 run.py --dataset text2image-10M --algorithm sustech-ood --neurips23track ood -``` - -#### OOD Algorithm: vamana - -``` -python install.py --neurips23track ood --algorithm vamana -python3 run.py --dataset text2image-10M --algorithm vamana --neurips23track ood -``` - -#### OOD Algorithm: pinecone-ood - -``` -python install.py --neurips23track ood --algorithm pinecone-ood -python run.py --neurips23track ood --algorithm pinecone-ood --dataset text2image-10M -``` - -#### OOD Algorithm: zilliz - -``` -python install.py --neurips23track ood --algorithm zilliz -python3 run.py --dataset text2image-10M --algorithm zilliz --neurips23track ood -``` - ### Streaming Track Prepare the track dataset by running the following command in the top-level directory of the repository: @@ -519,88 +285,12 @@ python create_dataset.py --dataset msturing-30M-clustered python -m benchmark.streaming.download_gt --runbook_file neurips23/streaming/final_runbook.yaml --dataset msturing-30M-clustered ``` -#### Streaming Algorithm: cufe - -``` -python install.py --neurips23track streaming --algorithm cufe -python3 run.py --dataset msturing-30M-clustered --algorithm cufe --neurips23track streaming --runbook_path neurips23/streaming/final_runbook.yaml -``` - -#### Streaming Algorithm: diskann - -``` -python install.py --neurips23track streaming --algorithm diskann -python3 run.py --dataset msturing-30M-clustered --algorithm diskann --neurips23track streaming --runbook_path neurips23/streaming/final_runbook.yaml -``` - -#### Streaming Algorithm: hwtl_sdu_anns_stream - -``` -python install.py --neurips23track streaming --algorithm hwtl_sdu_anns_stream -python3 run.py --dataset msturing-30M-clustered --algorithm hwtl_sdu_anns_stream --neurips23track streaming --runbook_path neurips23/streaming/final_runbook.yaml -``` - -#### Streaming Algorithm: puck - -``` -python install.py --neurips23track streaming --algorithm puck -python3 run.py --dataset msturing-30M-clustered --algorithm puck --neurips23track streaming --runbook_path neurips23/streaming/final_runbook.yaml -``` - -#### Streaming Algorithm: pyanns - -``` -python install.py --neurips23track streaming --algorithm pyanns -python3 run.py --dataset msturing-30M-clustered --algorithm pyanns --neurips23track streaming --runbook_path neurips23/streaming/final_runbook.yaml # error, see below -2024-08-27 19:54:57,239 - annb.cfb02d822e66 - ERROR - Container.wait for container cfb02d822e66 failed with exception -2024-08-27 19:54:57,239 - annb.cfb02d822e66 - ERROR - Invoked with ['--dataset', 'msturing-30M-clustered', '--algorithm', 'pyanns', '--module', 'neurips23.streaming.pyanns.pyanns', '--constructor', 'Pyanns', ' ---runs', '5', '--count', '10', '--neurips23track', 'streaming', '--runbook_path', 'neurips23/streaming/final_runbook.yaml', '["euclidean", {"R": 32, "L": 100, "insert_threads": 8, "consolidate_threads": 8}]', -'[{"Ls": 300, "T": 8}]', '[{"Ls": 400, "T": 8}]', '[{"Ls": 500, "T": 8}]', '[{"Ls": 600, "T": 8}]'] -Traceback (most recent call last): - File "/home/gwilliams/anaconda3/envs/bigann-latitude-m2-medium/lib/python3.10/site-packages/urllib3/response.py", line 748, in _error_catcher - yield - File "/home/gwilliams/anaconda3/envs/bigann-latitude-m2-medium/lib/python3.10/site-packages/urllib3/response.py", line 1206, in read_chunked - self._update_chunk_length() - File "/home/gwilliams/anaconda3/envs/bigann-latitude-m2-medium/lib/python3.10/site-packages/urllib3/response.py", line 1125, in _update_chunk_length - line = self._fp.fp.readline() # type: ignore[union-attr] - File "/home/gwilliams/anaconda3/envs/bigann-latitude-m2-medium/lib/python3.10/socket.py", line 705, in readinto - return self._sock.recv_into(b) -TimeoutError: timed out -``` - -#### Streaming Algorithm: scann - -``` -python install.py --neurips23track streaming --algorithm scann -python3 run.py --dataset msturing-30M-clustered --algorithm scann --neurips23track streaming --runbook_path neurips23/streaming/final_runbook.yaml -``` -#### Streaming Algorithm: pinecone - -``` -python install.py --neurips23track streaming --algorithm pinecone -python3 run.py --dataset msturing-30M-clustered --algorithm pinecone --neurips23track streaming --runbook_path neurips23/streaming/final_runbook.yaml # error, see below -2024-08-27 23:26:17,631 - annb.001a9034d319 - ERROR - Container.wait for container 001a9034d319 failed with exception -2024-08-27 23:26:17,631 - annb.001a9034d319 - ERROR - Invoked with ['--dataset', 'msturing-30M-clustered', '--algorithm', 'pinecone', '--module', 'neurips23.streaming.pinecone.pinecone', ' ---constructor', 'pinecone', '--runs', '5', '--count', '10', '--neurips23track', 'streaming', '--runbook_path', 'neurips23/streaming/final_runbook.yaml', '["euclidean", {"R": 32, "L": 100, -"insert_threads": 8, "consolidate_threads": 8}]', '[{"Ls": 300, "k_1": 30, "T": 8}]', '[{"Ls": 400, "k_1": 30, "T": 8}]', '[{"Ls": 500, "k_1": 30, "T": 8}]', '[{"Ls": 520, "k_1": 30, "T": -8}]', '[{"Ls": 540, "k_1": 30, "T": 8}]', '[{"Ls": 560, "k_1": 30, "T": 8}]'] -Traceback (most recent call last): - File "/home/gwilliams/anaconda3/envs/bigann-latitude-m2-medium/lib/python3.10/site-packages/urllib3/response.py", line 748, in _error_catcher - yield - File "/home/gwilliams/anaconda3/envs/bigann-latitude-m2-medium/lib/python3.10/site-packages/urllib3/response.py", line 1206, in read_chunked - self._update_chunk_length() - File "/home/gwilliams/anaconda3/envs/bigann-latitude-m2-medium/lib/python3.10/site-packages/urllib3/response.py", line 1125, in _update_chunk_length - line = self._fp.fp.readline() # type: ignore[union-attr] - File "/home/gwilliams/anaconda3/envs/bigann-latitude-m2-medium/lib/python3.10/socket.py", line 705, in readinto - return self._sock.recv_into(b) -TimeoutError: timed out -``` ### Analysis To extract the data as CSV: ``` -sudo chmod ugo+r -R ./results/ # recursively add read permissions to data files -python data_export.py --recompute --output neurips23/latitude/data_export_m2-medium.csv +sudo chmod ugo+rw -R ./results/ # recursively add read permissions to data files +python data_export.py --recompute --output neurips23/latitude/data_export_m4-metal-medium.csv ``` To plot individual tracks: @@ -608,9 +298,11 @@ To plot individual tracks: python plot.py --neurips23track sparse --output neurips23/latitude/sparse.png --raw --recompute --dataset sparse-full python plot.py --neurips23track filter --output neurips23/latitude/filter.png --raw --recompute --dataset yfcc-10M python plot.py --neurips23track ood --output neurips23/latitude/ood.png --raw --recompute --dataset text2image-10M -TODO: streaming +TODO: streaming track ``` +To render the ranking table, see the analysis.ipynb notebook in the 'latitude' subdirectory. + ## Disclaimers_And_Credits * The hardware systems were graciously donated by [Latitude](https://www.latitude.sh/) diff --git a/neurips23/latitude/_latitude-m4-metal-medium.md b/neurips23/latitude/_latitude-m4-metal-medium.md index a29b7dea..aea10ee0 100644 --- a/neurips23/latitude/_latitude-m4-metal-medium.md +++ b/neurips23/latitude/_latitude-m4-metal-medium.md @@ -13,7 +13,7 @@ The NeurIPS2023 Practical Vector Search Challenge evaluated participating algorithms on Azure and EC2 CPU-based hardware instances. -In pursuit of expanding the evaluation criteria, we are evaluating on other generally available hardware configurations. +In pursuit of expanding the evaluation criteria, we are also running on other generally available hardware configurations. Shown here are results run on the following hardware: * AMD EPYC 9124 16-Core 3GHz processor @@ -27,6 +27,11 @@ Note: ## Results +Notes: +* Evaluations were run in late August 2024 +* Click on algorithm link to see the build and run commmand used (and errors if any). +* Pareto graphs for each track shown below. + ### Track: Filter ![Filter](latitude/filter.png) @@ -41,7 +46,7 @@ Note: ### Track: Streaming -![Streaming](latitude/streaming.png) +TODO ### Data Export @@ -82,160 +87,12 @@ Prepare the track dataset by running the following command in the top-level dire python create_dataset.py --dataset sparse-full ``` -#### Sparse Algorithm: cufe - -``` -python install.py --neurips23track sparse --algorithm cufe -python -m pip install requests==2.31.0 # urllib3.exceptions.URLSchemeUnknown: Not supported URL scheme http+docker -``` - -#### Sparse Algorithm: linscan - -``` -python install.py --neurips23track sparse --algorithm linscan -python run.py --dataset sparse-full --algorithm linscan --neurips23track sparse -``` - -#### Sparse Algorithm: nle - -``` -python install.py --neurips23track sparse --algorithm nle -python run.py --dataset sparse-full --algorithm nle --neurips23track sparse -``` - -#### Sparse Algorithm: pyanns - -``` -python install.py --neurips23track sparse --algorithm pyanns -python run.py --dataset sparse-full --algorithm pyanns --neurips23track sparse -``` - -#### Sparse Algorithm: shnsw - -``` -python install.py --neurips23track sparse --algorithm shnsw -python run.py --dataset sparse-full --algorithm shnsw --neurips23track sparse -``` - -#### Sparse Algorithm: spmat - -``` -python install.py --neurips23track sparse --algorithm spmat -python run.py --dataset sparse-full --algorithm spmat --neurips23track sparse # ERROR: sparse-full is not in spmat/config.yaml -``` - -#### Sparse Algorithm: sustech-whu - -``` -python install.py --neurips23track sparse --algorithm sustech-whu -# ERROR: git could not clone 'https://github.com/lizzy-0323/SUSTech-WHU-Sparse.git' -``` - -### Sparse Algorithm: pinecone_smips - -``` -python install.py --neurips23track sparse --algorithm pinecone_smips -python run.py --dataset sparse-full --algorithm pinecone_smips --neurips23track sparse -``` - -#### Sparse Algorithm: zilliz - -``` -python install.py --neurips23track sparse --algorithm zilliz -python run.py --dataset sparse-full --algorithm zilliz --neurips23track sparse -``` - ### Filter Track Prepare the track dataset by running the following command in the top-level directory of the repository: ``` python create_dataset.py --dataset yfcc-10M ``` - -#### Filter Algorithm: cufe - -``` -python install.py --neurips23track filter --algorithm cufe -python3 run.py --dataset yfcc-10M --algorithm cufe --neurips23track filter -``` - -#### Filter Algorithm: dhq - -``` -python install.py --neurips23track filter --algorithm dhq -# ERROR: failed to solve: process "/bin/sh -c python3 -c 'import faiss; print(faiss.IDSelectorFilterWise); print(faiss.__version__)'" did not complete successfully: exit code: 1 -``` - -#### Filter Algorithm: faiss - -``` -python install.py --neurips23track filter --algorithm faiss -python3 run.py --dataset yfcc-10M --algorithm faiss --neurips23track filter -``` - -#### Filter Algorithm: faissplus - -``` -python install.py --neurips23track filter --algorithm faissplus -python3 run.py --dataset yfcc-10M --algorithm faissplus --neurips23track filter -``` - -#### Filter Algorithm: fdufilterdiskann - -``` -python install.py --neurips23track filter --algorithm fdufilterdiskann -# ERROR: failed to solve: process "/bin/sh -c git clone --recursive --branch main https://github.com/PUITAR/FduFilterDiskANN.git" did not complete successfully: exit code: 128 -``` - -#### Filter Algorithm: hwtl_sdu_anns_filter - -``` -python install.py --neurips23track filter --algorithm hwtl_sdu_anns_filter -# ERROR: failed to solve: process "/bin/sh -c python3 -c 'import faiss; print(faiss.IndexFlatL2); print(faiss.__version__)'" did not complete successfully: exit code: 1 -``` - -#### Filter Algorithm: parlayivf - -``` -python install.py --neurips23track filter --algorithm parlayivf -python3 run.py --dataset yfcc-10M --algorithm parlayivf --neurips23track filter -``` - -#### Filter Algorithm: puck - -``` -python install.py --neurips23track filter --algorithm puck -python3 run.py --dataset yfcc-10M --algorithm puck --neurips23track filter -``` - -#### Filter Algorithm: pyanns - -``` -python install.py --neurips23track filter --algorithm pyanns -python3 run.py --dataset yfcc-10M --algorithm pyanns --neurips23track filter -``` - -#### Filter Algorithm: wm_filter - -``` -python install.py --neurips23track filter --algorithm wm_filter -python3 run.py --dataset yfcc-10M --algorithm wm_filter --neurips23track filter -``` - -#### Filter Algorithm: pinecone - -``` -python install.py --neurips23track filter --algorithm pinecone -python run.py --neurips23track filter --algorithm pinecone --dataset yfcc-10M -``` - -#### Filter Algorithm: zilliz - -``` -python install.py --neurips23track filter --algorithm zilliz -python3 run.py --dataset yfcc-10M --algorithm zilliz --neurips23track filter -``` - ### OOD Track Prepare the track dataset by running the following command in the top-level directory of the repository: @@ -243,97 +100,6 @@ Prepare the track dataset by running the following command in the top-level dire python create_dataset.py --dataset text2image-10M ``` -#### OOD Algorithm: cufe - -``` -python install.py --neurips23track ood --algorithm cufe -python3 run.py --dataset text2image-10M --algorithm cufe --neurips23track ood -``` - -#### OOD Algorithm: diskann - -``` -python install.py --neurips23track ood --algorithm diskann -python3 run.py --dataset text2image-10M --algorithm diskann --neurips23track ood -``` - -#### OOD Algorithm: epsearch - -``` -python install.py --neurips23track ood --algorithm epsearch -python3 run.py --dataset text2image-10M --algorithm epsearch --neurips23track ood -``` - -#### OOD Algorithm: mysteryann - -``` -python install.py --neurips23track ood --algorithm mysteryann -python3 run.py --dataset text2image-10M --algorithm mysteryann --neurips23track ood -``` - -#### OOD Algorithm: mysteryann-dif - -``` -python install.py --neurips23track ood --algorithm mysteryann-dif -python3 run.py --dataset text2image-10M --algorithm mysteryann-dif --neurips23track ood -``` - -#### OOD Algorithm: ngt - -``` -python install.py --neurips23track ood --algorithm ngt -python3 run.py --dataset text2image-10M --algorithm ngt --neurips23track ood -``` - -#### OOD Algorithm: puck - -``` -python install.py --neurips23track ood --algorithm puck -python3 run.py --dataset text2image-10M --algorithm puck --neurips23track ood -``` - -#### OOD Algorithm: puck-fizz - -``` -python install.py --neurips23track ood --algorithm puck-fizz -ERROR: failed to solve: process "/bin/sh -c git clone -b ood-try https://github.com/baidu/puck.git" did not complete successfully: exit code: 128 -``` - -#### OOD Algorithm: pyanns - -``` -python install.py --neurips23track ood --algorithm pyanns -python3 run.py --dataset text2image-10M --algorithm pyanns --neurips23track ood -``` - -#### OOD Algorithm: sustech-ood - -``` -python install.py --neurips23track ood --algorithm sustech-ood -python3 run.py --dataset text2image-10M --algorithm sustech-ood --neurips23track ood -``` - -#### OOD Algorithm: vamana - -``` -python install.py --neurips23track ood --algorithm vamana -python3 run.py --dataset text2image-10M --algorithm vamana --neurips23track ood -``` - -#### OOD Algorithm: pinecone-ood - -``` -python install.py --neurips23track ood --algorithm pinecone-ood -python run.py --neurips23track ood --algorithm pinecone-ood --dataset text2image-10M -``` - -#### OOD Algorithm: zilliz - -``` -python install.py --neurips23track ood --algorithm zilliz -python3 run.py --dataset text2image-10M --algorithm zilliz --neurips23track ood -``` - ### Streaming Track Prepare the track dataset by running the following command in the top-level directory of the repository: @@ -342,88 +108,12 @@ python create_dataset.py --dataset msturing-30M-clustered python -m benchmark.streaming.download_gt --runbook_file neurips23/streaming/final_runbook.yaml --dataset msturing-30M-clustered ``` -#### Streaming Algorithm: cufe - -``` -python install.py --neurips23track streaming --algorithm cufe -python3 run.py --dataset msturing-30M-clustered --algorithm cufe --neurips23track streaming --runbook_path neurips23/streaming/final_runbook.yaml -``` - -#### Streaming Algorithm: diskann - -``` -python install.py --neurips23track streaming --algorithm diskann -python3 run.py --dataset msturing-30M-clustered --algorithm diskann --neurips23track streaming --runbook_path neurips23/streaming/final_runbook.yaml -``` - -#### Streaming Algorithm: hwtl_sdu_anns_stream - -``` -python install.py --neurips23track streaming --algorithm hwtl_sdu_anns_stream -python3 run.py --dataset msturing-30M-clustered --algorithm hwtl_sdu_anns_stream --neurips23track streaming --runbook_path neurips23/streaming/final_runbook.yaml -``` - -#### Streaming Algorithm: puck - -``` -python install.py --neurips23track streaming --algorithm puck -python3 run.py --dataset msturing-30M-clustered --algorithm puck --neurips23track streaming --runbook_path neurips23/streaming/final_runbook.yaml -``` - -#### Streaming Algorithm: pyanns - -``` -python install.py --neurips23track streaming --algorithm pyanns -python3 run.py --dataset msturing-30M-clustered --algorithm pyanns --neurips23track streaming --runbook_path neurips23/streaming/final_runbook.yaml # error, see below -2024-08-27 19:54:57,239 - annb.cfb02d822e66 - ERROR - Container.wait for container cfb02d822e66 failed with exception -2024-08-27 19:54:57,239 - annb.cfb02d822e66 - ERROR - Invoked with ['--dataset', 'msturing-30M-clustered', '--algorithm', 'pyanns', '--module', 'neurips23.streaming.pyanns.pyanns', '--constructor', 'Pyanns', ' ---runs', '5', '--count', '10', '--neurips23track', 'streaming', '--runbook_path', 'neurips23/streaming/final_runbook.yaml', '["euclidean", {"R": 32, "L": 100, "insert_threads": 8, "consolidate_threads": 8}]', -'[{"Ls": 300, "T": 8}]', '[{"Ls": 400, "T": 8}]', '[{"Ls": 500, "T": 8}]', '[{"Ls": 600, "T": 8}]'] -Traceback (most recent call last): - File "/home/gwilliams/anaconda3/envs/bigann-latitude-m2-medium/lib/python3.10/site-packages/urllib3/response.py", line 748, in _error_catcher - yield - File "/home/gwilliams/anaconda3/envs/bigann-latitude-m2-medium/lib/python3.10/site-packages/urllib3/response.py", line 1206, in read_chunked - self._update_chunk_length() - File "/home/gwilliams/anaconda3/envs/bigann-latitude-m2-medium/lib/python3.10/site-packages/urllib3/response.py", line 1125, in _update_chunk_length - line = self._fp.fp.readline() # type: ignore[union-attr] - File "/home/gwilliams/anaconda3/envs/bigann-latitude-m2-medium/lib/python3.10/socket.py", line 705, in readinto - return self._sock.recv_into(b) -TimeoutError: timed out -``` - -#### Streaming Algorithm: scann - -``` -python install.py --neurips23track streaming --algorithm scann -python3 run.py --dataset msturing-30M-clustered --algorithm scann --neurips23track streaming --runbook_path neurips23/streaming/final_runbook.yaml -``` -#### Streaming Algorithm: pinecone - -``` -python install.py --neurips23track streaming --algorithm pinecone -python3 run.py --dataset msturing-30M-clustered --algorithm pinecone --neurips23track streaming --runbook_path neurips23/streaming/final_runbook.yaml # error, see below -2024-08-27 23:26:17,631 - annb.001a9034d319 - ERROR - Container.wait for container 001a9034d319 failed with exception -2024-08-27 23:26:17,631 - annb.001a9034d319 - ERROR - Invoked with ['--dataset', 'msturing-30M-clustered', '--algorithm', 'pinecone', '--module', 'neurips23.streaming.pinecone.pinecone', ' ---constructor', 'pinecone', '--runs', '5', '--count', '10', '--neurips23track', 'streaming', '--runbook_path', 'neurips23/streaming/final_runbook.yaml', '["euclidean", {"R": 32, "L": 100, -"insert_threads": 8, "consolidate_threads": 8}]', '[{"Ls": 300, "k_1": 30, "T": 8}]', '[{"Ls": 400, "k_1": 30, "T": 8}]', '[{"Ls": 500, "k_1": 30, "T": 8}]', '[{"Ls": 520, "k_1": 30, "T": -8}]', '[{"Ls": 540, "k_1": 30, "T": 8}]', '[{"Ls": 560, "k_1": 30, "T": 8}]'] -Traceback (most recent call last): - File "/home/gwilliams/anaconda3/envs/bigann-latitude-m2-medium/lib/python3.10/site-packages/urllib3/response.py", line 748, in _error_catcher - yield - File "/home/gwilliams/anaconda3/envs/bigann-latitude-m2-medium/lib/python3.10/site-packages/urllib3/response.py", line 1206, in read_chunked - self._update_chunk_length() - File "/home/gwilliams/anaconda3/envs/bigann-latitude-m2-medium/lib/python3.10/site-packages/urllib3/response.py", line 1125, in _update_chunk_length - line = self._fp.fp.readline() # type: ignore[union-attr] - File "/home/gwilliams/anaconda3/envs/bigann-latitude-m2-medium/lib/python3.10/socket.py", line 705, in readinto - return self._sock.recv_into(b) -TimeoutError: timed out -``` ### Analysis To extract the data as CSV: ``` -sudo chmod ugo+r -R ./results/ # recursively add read permissions to data files -python data_export.py --recompute --output neurips23/latitude/data_export_m2-medium.csv +sudo chmod ugo+rw -R ./results/ # recursively add read permissions to data files +python data_export.py --recompute --output neurips23/latitude/data_export_m4-metal-medium.csv ``` To plot individual tracks: @@ -431,9 +121,11 @@ To plot individual tracks: python plot.py --neurips23track sparse --output neurips23/latitude/sparse.png --raw --recompute --dataset sparse-full python plot.py --neurips23track filter --output neurips23/latitude/filter.png --raw --recompute --dataset yfcc-10M python plot.py --neurips23track ood --output neurips23/latitude/ood.png --raw --recompute --dataset text2image-10M -TODO: streaming +TODO: streaming track ``` +To render the ranking table, see the analysis.ipynb notebook in the 'latitude' subdirectory. + ## Disclaimers_And_Credits * The hardware systems were graciously donated by [Latitude](https://www.latitude.sh/) diff --git a/neurips23/latitude/analysis.ipynb b/neurips23/latitude/analysis.ipynb index 752de002..fbe5d3a6 100644 --- a/neurips23/latitude/analysis.ipynb +++ b/neurips23/latitude/analysis.ipynb @@ -2,11 +2,13 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": 51, "id": "21ac7846-3ebf-4f2a-b545-be471d0ce5a8", "metadata": {}, "outputs": [], "source": [ + "# notebook config\n", + "\n", "# Relative path to competition data export file\n", "CSV = \"data_export_m4-metal-medium.csv\"\n", "\n", @@ -25,11 +27,13 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 52, "id": "936c9c86-f611-4b3f-ae6b-d9d02347a4e2", "metadata": {}, "outputs": [], "source": [ + "# package imports\n", + "\n", "import pandas as pd\n", "import os\n", "import glob\n", @@ -42,43 +46,49 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 53, "id": "071746d6-1cdc-404e-beef-d643596218fd", "metadata": {}, "outputs": [], "source": [ + "# global config\n", + "\n", "pd.set_option('display.precision', 2)" ] }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 54, "id": "7193be1a-a6de-43df-9f60-9243f5ecf473", "metadata": {}, "outputs": [], "source": [ "# read CSV\n", + "\n", "df = pd.read_csv( CSV )\n" ] }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 55, "id": "5c83a9db-fc3f-4813-91c8-a7a1275b086d", "metadata": {}, "outputs": [], "source": [ "# get track/dataset groups\n", + "\n", "grps = df.groupby([\"track\",\"dataset\"])" ] }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 56, "id": "303b19d1-d0a6-4f0c-9dd0-08879df9cdd6", "metadata": {}, "outputs": [], "source": [ + "# transform columns\n", + "\n", "dfs = [] # accumulate individual track dataframes\n", "\n", "# iterate groups\n", @@ -131,7 +141,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 57, "id": "51cf0a75-1acf-44de-b7f1-2cf030cadd94", "metadata": {}, "outputs": [ @@ -168,90 +178,90 @@ " \n", " \n", " 0\n", + " zilliz\n", + " 213.29\n", + " 0.93\n", + " ok\n", + " 1\n", + " filter\n", + " yfcc-10M\n", + " \n", + " \n", + " 1\n", " pinecone\n", " 146.72\n", " 0.92\n", " ok\n", - " 1\n", + " 2\n", " filter\n", " yfcc-10M\n", " \n", " \n", - " 1\n", + " 2\n", " puck\n", " 62.26\n", " 0.94\n", " ok\n", - " 2\n", + " 3\n", " filter\n", " yfcc-10M\n", " \n", " \n", - " 2\n", + " 3\n", " parlayivf\n", " 55.03\n", " 0.95\n", " ok\n", - " 3\n", + " 4\n", " filter\n", " yfcc-10M\n", " \n", " \n", - " 3\n", + " 4\n", " wm_filter\n", " 20.87\n", " 0.92\n", " ok\n", - " 4\n", + " 5\n", " filter\n", " yfcc-10M\n", " \n", " \n", - " 4\n", + " 5\n", " pyanns\n", " 8.99\n", " 0.91\n", " ok\n", - " 5\n", + " 6\n", " filter\n", " yfcc-10M\n", " \n", " \n", - " 5\n", + " 6\n", " faissplus\n", " 8.49\n", " 0.91\n", " ok\n", - " 6\n", + " 7\n", " filter\n", " yfcc-10M\n", " \n", " \n", - " 6\n", + " 7\n", " faiss\n", " 7.33\n", " 0.99\n", " ok\n", - " 7\n", + " 8\n", " filter\n", " yfcc-10M\n", " \n", " \n", - " 7\n", + " 8\n", " cufe\n", " 6.32\n", " 0.94\n", " ok\n", - " 8\n", - " filter\n", - " yfcc-10M\n", - " \n", - " \n", - " 8\n", - " fdufilterdiskann\n", - " NaN\n", - " NaN\n", - " error\n", " 9\n", " filter\n", " yfcc-10M\n", @@ -268,7 +278,7 @@ " \n", " \n", " 10\n", - " zilliz\n", + " fdufilterdiskann\n", " NaN\n", " NaN\n", " error\n", @@ -298,9 +308,9 @@ " \n", " \n", " 1\n", - " pyanns\n", - " 55.50\n", - " 0.92\n", + " zilliz\n", + " 73.51\n", + " 0.91\n", " ok\n", " 2\n", " ood\n", @@ -308,8 +318,8 @@ " \n", " \n", " 2\n", - " sustech-ood\n", - " 28.46\n", + " pyanns\n", + " 55.50\n", " 0.92\n", " ok\n", " 3\n", @@ -318,9 +328,9 @@ " \n", " \n", " 3\n", - " mysteryann-dif\n", - " 27.95\n", - " 0.91\n", + " scann\n", + " 32.26\n", + " 0.92\n", " ok\n", " 4\n", " ood\n", @@ -328,9 +338,9 @@ " \n", " \n", " 4\n", - " mysteryann\n", - " 26.56\n", - " 0.91\n", + " sustech-ood\n", + " 28.46\n", + " 0.92\n", " ok\n", " 5\n", " ood\n", @@ -338,9 +348,9 @@ " \n", " \n", " 5\n", - " vamana\n", - " 19.97\n", - " 0.94\n", + " mysteryann-dif\n", + " 27.95\n", + " 0.91\n", " ok\n", " 6\n", " ood\n", @@ -348,8 +358,8 @@ " \n", " \n", " 6\n", - " puck\n", - " 18.96\n", + " mysteryann\n", + " 26.56\n", " 0.91\n", " ok\n", " 7\n", @@ -358,9 +368,9 @@ " \n", " \n", " 7\n", - " ngt\n", - " 11.92\n", - " 0.95\n", + " vamana\n", + " 19.97\n", + " 0.94\n", " ok\n", " 8\n", " ood\n", @@ -368,9 +378,9 @@ " \n", " \n", " 8\n", - " epsearch\n", - " 7.66\n", - " 0.92\n", + " puck\n", + " 18.96\n", + " 0.91\n", " ok\n", " 9\n", " ood\n", @@ -378,9 +388,9 @@ " \n", " \n", " 9\n", - " diskann\n", - " 6.41\n", - " 0.91\n", + " ngt\n", + " 11.92\n", + " 0.95\n", " ok\n", " 10\n", " ood\n", @@ -388,9 +398,9 @@ " \n", " \n", " 10\n", - " cufe\n", - " 5.40\n", - " 0.91\n", + " epsearch\n", + " 7.66\n", + " 0.92\n", " ok\n", " 11\n", " ood\n", @@ -398,27 +408,27 @@ " \n", " \n", " 11\n", - " scann\n", - " NaN\n", - " NaN\n", - " error\n", + " diskann\n", + " 6.41\n", + " 0.91\n", + " ok\n", " 12\n", " ood\n", " text2image-10M\n", " \n", " \n", " 12\n", - " puck-fizz\n", - " NaN\n", - " NaN\n", - " error\n", + " cufe\n", + " 5.40\n", + " 0.91\n", + " ok\n", " 13\n", " ood\n", " text2image-10M\n", " \n", " \n", " 13\n", - " zilliz\n", + " puck-fizz\n", " NaN\n", " NaN\n", " error\n", @@ -428,77 +438,77 @@ " \n", " \n", " 0\n", + " zilliz\n", + " 34.76\n", + " 0.93\n", + " ok\n", + " 1\n", + " sparse\n", + " sparse-full\n", + " \n", + " \n", + " 1\n", " pyanns\n", " 26.86\n", " 0.92\n", " ok\n", - " 1\n", + " 2\n", " sparse\n", " sparse-full\n", " \n", " \n", - " 1\n", + " 2\n", " pinecone_smips\n", " 12.00\n", " 0.90\n", " ok\n", - " 2\n", + " 3\n", " sparse\n", " sparse-full\n", " \n", " \n", - " 2\n", + " 3\n", " shnsw\n", " 8.25\n", " 0.94\n", " ok\n", - " 3\n", + " 4\n", " sparse\n", " sparse-full\n", " \n", " \n", - " 3\n", + " 4\n", " nle\n", " 2.95\n", " 0.94\n", " ok\n", - " 4\n", + " 5\n", " sparse\n", " sparse-full\n", " \n", " \n", - " 4\n", + " 5\n", " cufe\n", " 0.08\n", " 0.99\n", " ok\n", - " 5\n", + " 6\n", " sparse\n", " sparse-full\n", " \n", " \n", - " 5\n", + " 6\n", " linscan\n", " 0.06\n", " 0.98\n", " ok\n", - " 6\n", - " sparse\n", - " sparse-full\n", - " \n", - " \n", - " 6\n", - " sustech-whu\n", - " NaN\n", - " NaN\n", - " error\n", " 7\n", " sparse\n", " sparse-full\n", " \n", " \n", " 7\n", - " zilliz\n", + " sustech-whu\n", " NaN\n", " NaN\n", " error\n", @@ -522,40 +532,40 @@ ], "text/plain": [ " algorithm qps recall/ap status rank track \\\n", - "0 pinecone 146.72 0.92 ok 1 filter \n", - "1 puck 62.26 0.94 ok 2 filter \n", - "2 parlayivf 55.03 0.95 ok 3 filter \n", - "3 wm_filter 20.87 0.92 ok 4 filter \n", - "4 pyanns 8.99 0.91 ok 5 filter \n", - "5 faissplus 8.49 0.91 ok 6 filter \n", - "6 faiss 7.33 0.99 ok 7 filter \n", - "7 cufe 6.32 0.94 ok 8 filter \n", - "8 fdufilterdiskann NaN NaN error 9 filter \n", + "0 zilliz 213.29 0.93 ok 1 filter \n", + "1 pinecone 146.72 0.92 ok 2 filter \n", + "2 puck 62.26 0.94 ok 3 filter \n", + "3 parlayivf 55.03 0.95 ok 4 filter \n", + "4 wm_filter 20.87 0.92 ok 5 filter \n", + "5 pyanns 8.99 0.91 ok 6 filter \n", + "6 faissplus 8.49 0.91 ok 7 filter \n", + "7 faiss 7.33 0.99 ok 8 filter \n", + "8 cufe 6.32 0.94 ok 9 filter \n", "9 hwtl_sdu_anns_filter NaN NaN error 10 filter \n", - "10 zilliz NaN NaN error 11 filter \n", + "10 fdufilterdiskann NaN NaN error 11 filter \n", "11 dhq NaN NaN error 12 filter \n", "0 pinecone-ood 76.87 0.91 ok 1 ood \n", - "1 pyanns 55.50 0.92 ok 2 ood \n", - "2 sustech-ood 28.46 0.92 ok 3 ood \n", - "3 mysteryann-dif 27.95 0.91 ok 4 ood \n", - "4 mysteryann 26.56 0.91 ok 5 ood \n", - "5 vamana 19.97 0.94 ok 6 ood \n", - "6 puck 18.96 0.91 ok 7 ood \n", - "7 ngt 11.92 0.95 ok 8 ood \n", - "8 epsearch 7.66 0.92 ok 9 ood \n", - "9 diskann 6.41 0.91 ok 10 ood \n", - "10 cufe 5.40 0.91 ok 11 ood \n", - "11 scann NaN NaN error 12 ood \n", - "12 puck-fizz NaN NaN error 13 ood \n", - "13 zilliz NaN NaN error 14 ood \n", - "0 pyanns 26.86 0.92 ok 1 sparse \n", - "1 pinecone_smips 12.00 0.90 ok 2 sparse \n", - "2 shnsw 8.25 0.94 ok 3 sparse \n", - "3 nle 2.95 0.94 ok 4 sparse \n", - "4 cufe 0.08 0.99 ok 5 sparse \n", - "5 linscan 0.06 0.98 ok 6 sparse \n", - "6 sustech-whu NaN NaN error 7 sparse \n", - "7 zilliz NaN NaN error 8 sparse \n", + "1 zilliz 73.51 0.91 ok 2 ood \n", + "2 pyanns 55.50 0.92 ok 3 ood \n", + "3 scann 32.26 0.92 ok 4 ood \n", + "4 sustech-ood 28.46 0.92 ok 5 ood \n", + "5 mysteryann-dif 27.95 0.91 ok 6 ood \n", + "6 mysteryann 26.56 0.91 ok 7 ood \n", + "7 vamana 19.97 0.94 ok 8 ood \n", + "8 puck 18.96 0.91 ok 9 ood \n", + "9 ngt 11.92 0.95 ok 10 ood \n", + "10 epsearch 7.66 0.92 ok 11 ood \n", + "11 diskann 6.41 0.91 ok 12 ood \n", + "12 cufe 5.40 0.91 ok 13 ood \n", + "13 puck-fizz NaN NaN error 14 ood \n", + "0 zilliz 34.76 0.93 ok 1 sparse \n", + "1 pyanns 26.86 0.92 ok 2 sparse \n", + "2 pinecone_smips 12.00 0.90 ok 3 sparse \n", + "3 shnsw 8.25 0.94 ok 4 sparse \n", + "4 nle 2.95 0.94 ok 5 sparse \n", + "5 cufe 0.08 0.99 ok 6 sparse \n", + "6 linscan 0.06 0.98 ok 7 sparse \n", + "7 sustech-whu NaN NaN error 8 sparse \n", "8 spmat NaN NaN error 9 sparse \n", "\n", " dataset \n", @@ -596,7 +606,7 @@ "8 sparse-full " ] }, - "execution_count": 20, + "execution_count": 57, "metadata": {}, "output_type": "execute_result" } @@ -609,12 +619,13 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 62, "id": "c90fba89-728a-4f54-bb80-95834bd33005", "metadata": {}, "outputs": [], "source": [ "# pivot via 'rank' so that track rankings are parallel across columns\n", + "\n", "pivot_df = master_df.pivot_table(index=['rank'], \n", " columns=['track'], \n", " values=['algorithm','qps','status'],\n", @@ -624,12 +635,13 @@ }, { "cell_type": "code", - "execution_count": 41, + "execution_count": 63, "id": "f4e617b4-112b-45e1-bc3b-2692d980d19f", "metadata": {}, "outputs": [], "source": [ "# group specific track columns\n", + "\n", "reorder_df = pivot_df[ [ ('algorithm','filter'), ( 'qps','filter' ), \\\n", " ('algorithm','sparse'), ( 'qps','sparse' ), \\\n", " ('algorithm','ood'), ( 'qps','ood' ) ] ]\n", @@ -638,24 +650,26 @@ }, { "cell_type": "code", - "execution_count": 42, + "execution_count": 64, "id": "8d2653b4-f070-4761-8b5e-9a9fa03f19cc", "metadata": {}, "outputs": [], "source": [ "# swap hierarchical index for columns\n", + "\n", "swap_df = reorder_df.swaplevel(0,1, axis=1)\n", "#swap_df" ] }, { "cell_type": "code", - "execution_count": 43, + "execution_count": 65, "id": "d0accb53-c34b-418e-85a4-253c46cdb90b", "metadata": {}, "outputs": [], "source": [ "# add per track 'rank' column ensuring no rank to algorithms that did not qualify\n", + "\n", "for track in master_df['track'].unique():\n", " qps = list( swap_df[ (track,'qps') ] )\n", " track_ranking = map(lambda el: float('nan') if math.isnan(qps[el-1]) else el,\n", @@ -666,12 +680,13 @@ "\n", " swap_df.insert(col_idx, (track,'rank'), list(track_ranking) )\n", " swap_df[(track,'rank')] = swap_df[(track,'rank')].astype('Int64')\n", + " \n", "#swap_df" ] }, { "cell_type": "code", - "execution_count": 44, + "execution_count": 66, "id": "5cee1395-0e41-4ba4-b586-f9c4b56a9024", "metadata": {}, "outputs": [ @@ -680,25 +695,25 @@ "text/html": [ "\n", - "\n", + "
\n", " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", " \n", @@ -715,181 +730,181 @@ " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", "
trackfiltersparseoodfiltersparseood
 rankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqps
rank
11pinecone146.7212421pyanns26.8585191pinecone-ood76.865423
22puck62.2576952pinecone_smips12.0031822pyanns55.504352
33parlayivf55.0331983shnsw8.2472773sustech-ood28.458263
44wm_filter20.8748574nle2.9456264mysteryann-dif27.946097
55pyanns8.9880225cufe0.0849765mysteryann26.560867
66faissplus8.4932646linscan0.0607566vamana19.965517
77faiss7.327855sustech-whunan7puck18.960224
88cufe6.318462zilliznan8ngt11.921399
9fdufilterdiskannnanspmatnan9epsearch7.663219
10hwtl_sdu_anns_filternannannan10diskann6.406821
11zilliznannannan11cufe5.404492
12dhqnannannanscannnan
13nannannannanpuck-fizznan
14nannannannanzilliznan11zilliz213.2850591zilliz34.7599761pinecone-ood76.865423
22pinecone146.7212422pyanns26.8585192zilliz73.509610
33puck62.2576953pinecone_smips12.0031823pyanns55.504352
44parlayivf55.0331984shnsw8.2472774scann32.256046
55wm_filter20.8748575nle2.9456265sustech-ood28.458263
66pyanns8.9880226cufe0.0849766mysteryann-dif27.946097
77faissplus8.4932647linscan0.0607567mysteryann26.560867
88faiss7.327855sustech-whunan8vamana19.965517
99cufe6.318462spmatnan9puck18.960224
10hwtl_sdu_anns_filternannannan10ngt11.921399
11fdufilterdiskannnannannan11epsearch7.663219
12dhqnannannan12diskann6.406821
13nannannannan13cufe5.404492
14nannannannanpuck-fizznan
\n" ], "text/plain": [ - "" + "" ] }, - "execution_count": 44, + "execution_count": 66, "metadata": {}, "output_type": "execute_result" } @@ -903,186 +918,200 @@ }, { "cell_type": "code", - "execution_count": 45, + "execution_count": 67, "id": "884f5113-01a9-42a7-9262-5cea8e9cd751", "metadata": {}, "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "WARNING: Using error file path for sparse/zilliz\n", + "WARNING: Using error file path for ood/scann\n", + "WARNING: Using error file path for sparse/sustech-whu\n", + "WARNING: Using error file path for sparse/spmat\n", + "WARNING: Using error file path for filter/hwtl_sdu_anns_filter\n", + "WARNING: Using error file path for filter/fdufilterdiskann\n", + "WARNING: Using error file path for filter/dhq\n", + "WARNING: Using error file path for ood/puck-fizz\n" + ] + }, { "data": { "text/html": [ - "\n", + "
\n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", "
filtersparseoodfiltersparseood
rankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqps
1
pinecone
 146.7K1
pyanns
 26.9K1
pinecone-ood
 76.9K
2
puck
 62.3K2
pinecone_smips
 12.0K2
pyanns
 55.5K
3
parlayivf
 55.0K3
shnsw
 8.2K3
sustech-ood
 28.5K
4
wm_filter
 20.9K4
nle
 2.9K4
mysteryann-dif
 27.9K
5
pyanns
 9.0K5
cufe
 0.1K5
mysteryann
 26.6K
6
faissplus
 8.5K6
linscan
 0.1K6
vamana
 20.0K
7
faiss
 7.3K
sustech-whu
7
puck
 19.0K
8
cufe
 6.3K
zilliz
8
ngt
 11.9K
fdufilterdiskann
spmat
9
epsearch
 7.7K
hwtl_sdu_anns_filter
10
diskann
 6.4K
zilliz
11
cufe
 5.4K
dhq
scann
puck-fizz
zilliz
1
zilliz
 213.3K1
zilliz
 34.8K1
pinecone-ood
 76.9K
2
pinecone
 146.7K2
pyanns
 26.9K2
zilliz
 73.5K
3
puck
 62.3K3
pinecone_smips
 12.0K3
pyanns
 55.5K
4
parlayivf
 55.0K4
shnsw
 8.2K4
scann
 32.3K
5
wm_filter
 20.9K5
nle
 2.9K5
sustech-ood
 28.5K
6
pyanns
 9.0K6
cufe
 0.1K6
mysteryann-dif
 27.9K
7
faissplus
 8.5K7
linscan
 0.1K7
mysteryann
 26.6K
8
faiss
 7.3K
sustech-whu
8
vamana
 20.0K
9
cufe
 6.3K
spmat
9
puck
 19.0K
hwtl_sdu_anns_filter
10
ngt
 11.9K
fdufilterdiskann
11
epsearch
 7.7K
dhq
12
diskann
 6.4K
13
cufe
 5.4K
puck-fizz
\n" @@ -1091,7 +1120,7 @@ "" ] }, - "execution_count": 45, + "execution_count": 67, "metadata": {}, "output_type": "execute_result" } @@ -1127,7 +1156,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 69, "id": "25fc371b-1a14-4fca-97de-acd79070c0ad", "metadata": {}, "outputs": [ @@ -1135,7 +1164,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Wrote ../latitude-m2-medium.md\n" + "Wrote .././latitude-m4-metal-medium.md\n" ] } ], @@ -1154,19 +1183,12 @@ "#print(new_contents[0:1000])\n", "\n", "# update the markdown\n", - "with open(MARKDOWN,'w') as md:\n", + "fname = os.path.join( \"..\", MARKDOWN.replace(\"_\",\"\") )\n", + "with open( fname, 'w') as md:\n", " contents = md.write(new_contents)\n", "\n", - "print(\"Wrote\", MARKDOWN)" + "print(\"Wrote\", fname)" ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3f6622a5-db56-4eeb-b254-5002ed0ea098", - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": {