diff --git a/neurips23/latitude-m4-metal-medium.md b/neurips23/latitude-m4-metal-medium.md index a9b342a0..e3ba04dc 100644 --- a/neurips23/latitude-m4-metal-medium.md +++ b/neurips23/latitude-m4-metal-medium.md @@ -1,179 +1,179 @@ # Eval On AMD 3GHz/16-Core + 125GB RAM + NVMe SSD (Bare Metal) - +
- - - + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + +
filtersparseoodfiltersparseood
rankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqps
1
zilliz
 213.3K1
zilliz
 34.8K1
pinecone-ood
 76.9K1
zilliz
 213.3K1
zilliz
 34.8K1
pinecone-ood
 76.9K
2
pinecone
 146.7K2
pyanns
 26.9K2
zilliz
 73.5K2
pinecone
 146.7K2
pyanns
 26.9K2
zilliz
 73.5K
3
puck
 62.3K3
pinecone_smips
 12.0K3
pyanns
 55.5K3
puck
 62.3K3
pinecone_smips
 12.0K3
pyanns
 55.5K
4
parlayivf
 55.0K4
shnsw
 8.2K4
scann
 32.3K4
parlayivf
 55.0K4
shnsw
 8.2K4
scann
 32.3K
5
wm_filter
 20.9K5
nle
 2.9K5
sustech-ood
 28.5K5
wm_filter
 20.9K5
nle
 2.9K5
sustech-ood
 28.5K
6
pyanns
 9.0K6
cufe
 0.1K6
mysteryann-dif
 27.9K6
pyanns
 9.0K6
cufe
 0.1K6
mysteryann-dif
 27.9K
7
faissplus
 8.5K7
linscan
 0.1K7
mysteryann
 26.6K7
faissplus
 8.5K7
linscan
 0.1K7
mysteryann
 26.6K
8
faiss
 7.3K
sustech-whu
8
vamana
 20.0K8
faiss
 7.3K
sustech-whu
8
vamana
 20.0K
9
cufe
 6.3K
spmat
9
puck
 19.0K9
cufe
 6.3K
spmat
9
puck
 19.0K
fdufilterdiskann
10
ngt
 11.9K
hwtl_sdu_anns_filter
10
ngt
 11.9K
dhq
11
epsearch
 7.7K
fdufilterdiskann
11
epsearch
 7.7K
hwtl_sdu_anns_filter
12
diskann
 6.4K
dhq
12
diskann
 6.4K
13
cufe
 5.4K13
cufe
 5.4K
puck-fizz
puck-fizz
@@ -205,6 +205,7 @@ The calculated rankings are shown at the top. Notes: * Evaluations were run in late August 2024 +# All participating algorithms are shown, but only qualifying algorithms are ranked. * In the table, each track algorithm links to the build and run commmand used (and errors if any). * Pareto graphs for each track shown below. diff --git a/neurips23/latitude/_latitude-m4-metal-medium.md b/neurips23/latitude/_latitude-m4-metal-medium.md index f284345c..9f04f9b3 100644 --- a/neurips23/latitude/_latitude-m4-metal-medium.md +++ b/neurips23/latitude/_latitude-m4-metal-medium.md @@ -27,6 +27,7 @@ The calculated rankings are shown at the top. Notes: * Evaluations were run in late August 2024 +# All participating algorithms are shown, but only qualifying algorithms are ranked. * In the table, each track algorithm links to the build and run commmand used (and errors if any). * Pareto graphs for each track shown below. diff --git a/neurips23/latitude/analysis.ipynb b/neurips23/latitude/analysis.ipynb index 7e645d72..4f35edd3 100644 --- a/neurips23/latitude/analysis.ipynb +++ b/neurips23/latitude/analysis.ipynb @@ -268,7 +268,7 @@ " \n", " \n", " 9\n", - " hwtl_sdu_anns_filter\n", + " fdufilterdiskann\n", " NaN\n", " NaN\n", " error\n", @@ -278,7 +278,7 @@ " \n", " \n", " 10\n", - " fdufilterdiskann\n", + " dhq\n", " NaN\n", " NaN\n", " error\n", @@ -288,7 +288,7 @@ " \n", " \n", " 11\n", - " dhq\n", + " hwtl_sdu_anns_filter\n", " NaN\n", " NaN\n", " error\n", @@ -541,9 +541,9 @@ "6 faissplus 8.49 0.91 ok 7 filter \n", "7 faiss 7.33 0.99 ok 8 filter \n", "8 cufe 6.32 0.94 ok 9 filter \n", - "9 hwtl_sdu_anns_filter NaN NaN error 10 filter \n", - "10 fdufilterdiskann NaN NaN error 11 filter \n", - "11 dhq NaN NaN error 12 filter \n", + "9 fdufilterdiskann NaN NaN error 10 filter \n", + "10 dhq NaN NaN error 11 filter \n", + "11 hwtl_sdu_anns_filter NaN NaN error 12 filter \n", "0 pinecone-ood 76.87 0.91 ok 1 ood \n", "1 zilliz 73.51 0.91 ok 2 ood \n", "2 pyanns 55.50 0.92 ok 3 ood \n", @@ -976,7 +976,7 @@ " \n", " \n", " 9\n", - " hwtl_sdu_anns_filter\n", + " fdufilterdiskann\n", " NaN\n", " NaN\n", " error\n", @@ -986,7 +986,7 @@ " \n", " \n", " 10\n", - " fdufilterdiskann\n", + " dhq\n", " NaN\n", " NaN\n", " error\n", @@ -996,7 +996,7 @@ " \n", " \n", " 11\n", - " dhq\n", + " hwtl_sdu_anns_filter\n", " NaN\n", " NaN\n", " error\n", @@ -1042,9 +1042,9 @@ "6 faissplus 8.49 0.91 ok 7 filter \n", "7 faiss 7.33 0.99 ok 8 filter \n", "8 cufe 6.32 0.94 ok 9 filter \n", - "9 hwtl_sdu_anns_filter NaN NaN error 10 filter \n", - "10 fdufilterdiskann NaN NaN error 11 filter \n", - "11 dhq NaN NaN error 12 filter \n", + "9 fdufilterdiskann NaN NaN error 10 filter \n", + "10 dhq NaN NaN error 11 filter \n", + "11 hwtl_sdu_anns_filter NaN NaN error 12 filter \n", "\n", " dataset \n", "0 sparse-full \n", @@ -1174,25 +1174,25 @@ "text/html": [ "\n", - "\n", + "
\n", " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", " \n", @@ -1209,178 +1209,178 @@ " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", "
trackfiltersparseoodfiltersparseood
 rankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqps
rank
11zilliz213.2850591zilliz34.7599761pinecone-ood76.865423
22pinecone146.7212422pyanns26.8585192zilliz73.509610
33puck62.2576953pinecone_smips12.0031823pyanns55.504352
44parlayivf55.0331984shnsw8.2472774scann32.256046
55wm_filter20.8748575nle2.9456265sustech-ood28.458263
66pyanns8.9880226cufe0.0849766mysteryann-dif27.946097
77faissplus8.4932647linscan0.0607567mysteryann26.560867
88faiss7.327855sustech-whunan8vamana19.965517
99cufe6.318462spmatnan9puck18.960224
10hwtl_sdu_anns_filternannannan10ngt11.921399
11fdufilterdiskannnannannan11epsearch7.663219
12dhqnannannan12diskann6.406821
13nannannannan13cufe5.404492
14nannannannanpuck-fizznan11zilliz213.2850591zilliz34.7599761pinecone-ood76.865423
22pinecone146.7212422pyanns26.8585192zilliz73.509610
33puck62.2576953pinecone_smips12.0031823pyanns55.504352
44parlayivf55.0331984shnsw8.2472774scann32.256046
55wm_filter20.8748575nle2.9456265sustech-ood28.458263
66pyanns8.9880226cufe0.0849766mysteryann-dif27.946097
77faissplus8.4932647linscan0.0607567mysteryann26.560867
88faiss7.327855sustech-whunan8vamana19.965517
99cufe6.318462spmatnan9puck18.960224
10fdufilterdiskannnannannan10ngt11.921399
11dhqnannannan11epsearch7.663219
12hwtl_sdu_anns_filternannannan12diskann6.406821
13nannannannan13cufe5.404492
14nannannannanpuck-fizznan
\n" ], "text/plain": [ - "" + "" ] }, "execution_count": 13, @@ -1407,188 +1407,188 @@ "text": [ "WARNING: Using error file path for sparse/sustech-whu\n", "WARNING: Using error file path for sparse/spmat\n", - "WARNING: Using error file path for filter/hwtl_sdu_anns_filter\n", "WARNING: Using error file path for filter/fdufilterdiskann\n", "WARNING: Using error file path for filter/dhq\n", + "WARNING: Using error file path for filter/hwtl_sdu_anns_filter\n", "WARNING: Using error file path for ood/puck-fizz\n" ] }, { "data": { "text/html": [ - "\n", + "
\n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", "
filtersparseoodfiltersparseood
rankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqps
1
zilliz
 213.3K1
zilliz
 34.8K1
pinecone-ood
 76.9K
2
pinecone
 146.7K2
pyanns
 26.9K2
zilliz
 73.5K
3
puck
 62.3K3
pinecone_smips
 12.0K3
pyanns
 55.5K
4
parlayivf
 55.0K4
shnsw
 8.2K4
scann
 32.3K
5
wm_filter
 20.9K5
nle
 2.9K5
sustech-ood
 28.5K
6
pyanns
 9.0K6
cufe
 0.1K6
mysteryann-dif
 27.9K
7
faissplus
 8.5K7
linscan
 0.1K7
mysteryann
 26.6K
8
faiss
 7.3K
sustech-whu
8
vamana
 20.0K
9
cufe
 6.3K
spmat
9
puck
 19.0K
hwtl_sdu_anns_filter
10
ngt
 11.9K
fdufilterdiskann
11
epsearch
 7.7K
dhq
12
diskann
 6.4K
13
cufe
 5.4K
puck-fizz
1
zilliz
 213.3K1
zilliz
 34.8K1
pinecone-ood
 76.9K
2
pinecone
 146.7K2
pyanns
 26.9K2
zilliz
 73.5K
3
puck
 62.3K3
pinecone_smips
 12.0K3
pyanns
 55.5K
4
parlayivf
 55.0K4
shnsw
 8.2K4
scann
 32.3K
5
wm_filter
 20.9K5
nle
 2.9K5
sustech-ood
 28.5K
6
pyanns
 9.0K6
cufe
 0.1K6
mysteryann-dif
 27.9K
7
faissplus
 8.5K7
linscan
 0.1K7
mysteryann
 26.6K
8
faiss
 7.3K
sustech-whu
8
vamana
 20.0K
9
cufe
 6.3K
spmat
9
puck
 19.0K
fdufilterdiskann
10
ngt
 11.9K
dhq
11
epsearch
 7.7K
hwtl_sdu_anns_filter
12
diskann
 6.4K
13
cufe
 5.4K
puck-fizz
\n" @@ -1638,14 +1638,10 @@ "metadata": {}, "outputs": [ { - "ename": "TypeError", - "evalue": "bad operand type for unary +: 'str'", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[15], line 11\u001b[0m\n\u001b[1;32m 8\u001b[0m matches \u001b[38;5;241m=\u001b[39m re\u001b[38;5;241m.\u001b[39msearch(regexpr, contents, re\u001b[38;5;241m.\u001b[39mM)\n\u001b[1;32m 10\u001b[0m \u001b[38;5;66;03m# insert table html\u001b[39;00m\n\u001b[0;32m---> 11\u001b[0m new_contents \u001b[38;5;241m=\u001b[39m contents[\u001b[38;5;241m0\u001b[39m:matches\u001b[38;5;241m.\u001b[39mspan(\u001b[38;5;241m0\u001b[39m)[\u001b[38;5;241m1\u001b[39m]] \u001b[38;5;241m+\u001b[39m \u001b[38;5;241;43m+\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;130;43;01m\\n\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m \u001b[38;5;241m+\u001b[39m new_html \u001b[38;5;241m+\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m+\u001b[39m contents[matches\u001b[38;5;241m.\u001b[39mspan(\u001b[38;5;241m0\u001b[39m)[\u001b[38;5;241m1\u001b[39m]:]\n\u001b[1;32m 12\u001b[0m \u001b[38;5;66;03m#print(new_contents[0:1000])\u001b[39;00m\n\u001b[1;32m 13\u001b[0m \n\u001b[1;32m 14\u001b[0m \u001b[38;5;66;03m# update the markdown\u001b[39;00m\n\u001b[1;32m 15\u001b[0m fname \u001b[38;5;241m=\u001b[39m os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mjoin( \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m..\u001b[39m\u001b[38;5;124m\"\u001b[39m, MARKDOWN\u001b[38;5;241m.\u001b[39mreplace(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m_\u001b[39m\u001b[38;5;124m\"\u001b[39m,\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m\"\u001b[39m) )\n", - "\u001b[0;31mTypeError\u001b[0m: bad operand type for unary +: 'str'" + "name": "stdout", + "output_type": "stream", + "text": [ + "Wrote .././latitude-m4-metal-medium.md\n" ] } ],