diff --git a/neurips23/latitude-m4-metal-medium.md b/neurips23/latitude-m4-metal-medium.md index 031e2a07..dde7af52 100644 --- a/neurips23/latitude-m4-metal-medium.md +++ b/neurips23/latitude-m4-metal-medium.md @@ -1,179 +1,179 @@ # Eval On AMD 3GHz/16-Core + 125GB RAM + NVMe SSD (Bare Metal) - +
- - - + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + +
filtersparseoodfiltersparseood
rankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqps
1
zilliz
 213.3K1
zilliz
 34.8K1
pinecone-ood
 76.9K1
zilliz
 213.3K1
zilliz
 34.8K1
scann
 107.4K
2
pinecone
 146.7K2
pyanns
 26.9K2
zilliz
 73.5K2
pinecone
 146.7K2
pyanns
 26.9K2
pinecone-ood
 76.9K
3
puck
 62.3K3
pinecone_smips
 12.0K3
pyanns
 55.5K3
puck
 62.3K3
pinecone_smips
 12.0K3
zilliz
 73.5K
4
parlayivf
 55.0K4
shnsw
 8.2K4
scann
 32.3K4
parlayivf
 55.0K4
shnsw
 8.2K4
pyanns
 55.5K
5
wm_filter
 20.9K5
nle
 2.9K5
sustech-ood
 28.5K5
wm_filter
 20.9K5
nle
 2.9K5
sustech-ood
 28.5K
6
pyanns
 9.0K6
cufe
 0.1K6
mysteryann-dif
 27.9K6
pyanns
 9.0K6
cufe
 0.1K6
mysteryann-dif
 27.9K
7
faissplus
 8.5K7
linscan
 0.1K7
mysteryann
 26.6K7
faissplus
 8.5K7
linscan
 0.1K7
mysteryann
 26.6K
8
faiss
 7.3K
sustech-whu
8
vamana
 20.0K8
faiss
 7.3K
spmat
8
vamana
 20.0K
9
cufe
 6.3K
spmat
9
puck
 19.0K9
cufe
 6.3K
sustech-whu
9
puck
 19.0K
dhq
10
ngt
 11.9K
dhq
10
ngt
 11.9K
fdufilterdiskann
11
epsearch
 7.7K
fdufilterdiskann
11
epsearch
 7.7K
hwtl_sdu_anns_filter
12
diskann
 6.4K
hwtl_sdu_anns_filter
12
diskann
 6.3K
13
cufe
 5.4K13
cufe
 5.4K
puck-fizz
puck-fizz
diff --git a/neurips23/latitude/analysis.ipynb b/neurips23/latitude/analysis.ipynb index 524e519c..ac38aa31 100644 --- a/neurips23/latitude/analysis.ipynb +++ b/neurips23/latitude/analysis.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": 31, "id": "21ac7846-3ebf-4f2a-b545-be471d0ce5a8", "metadata": {}, "outputs": [], @@ -19,12 +19,15 @@ "RECALL_AP_THRESHOLD = 0.9\n", "\n", "# Markdown template file\n", - "MARKDOWN = \"./_latitude-m4-metal-medium.md\"" + "MARKDOWN = \"./_latitude-m4-metal-medium.md\"\n", + "\n", + "# Algorithms to ignore - mostly to deal with transient issues like new algorithms not yet merged\n", + "IGNORE_ALGOS=[\"hanns\"] " ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 32, "id": "936c9c86-f611-4b3f-ae6b-d9d02347a4e2", "metadata": {}, "outputs": [], @@ -43,7 +46,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 33, "id": "071746d6-1cdc-404e-beef-d643596218fd", "metadata": {}, "outputs": [], @@ -55,7 +58,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 34, "id": "7193be1a-a6de-43df-9f60-9243f5ecf473", "metadata": {}, "outputs": [], @@ -67,7 +70,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 35, "id": "5c83a9db-fc3f-4813-91c8-a7a1275b086d", "metadata": {}, "outputs": [], @@ -79,7 +82,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 37, "id": "303b19d1-d0a6-4f0c-9dd0-08879df9cdd6", "metadata": {}, "outputs": [], @@ -113,6 +116,9 @@ " # retrieve all participating track algorithm names via track algo subdirectory \n", " track_dir = os.path.join( COMPETITION_DIR, \"%s/*/Dockerfile\" % track )\n", " algos_participating = [os.path.basename(os.path.dirname(p)) for p in glob.glob( track_dir ) ]\n", + "\n", + " # remove any temp ignores\n", + " algos_participating = list( set(algos_participating ) - set( IGNORE_ALGOS ) ) \n", " \n", " # compute difference of algo lists of track subdirs and algos in results - these did not qualify\n", " algos_did_not_qualify = list( set(algos_participating) - set(list(ranking_df[\"algorithm\"])) )\n", @@ -138,7 +144,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 38, "id": "51cf0a75-1acf-44de-b7f1-2cf030cadd94", "metadata": {}, "outputs": [ @@ -265,7 +271,7 @@ " \n", " \n", " 9\n", - " fdufilterdiskann\n", + " dhq\n", " NaN\n", " NaN\n", " error\n", @@ -275,7 +281,7 @@ " \n", " \n", " 10\n", - " dhq\n", + " fdufilterdiskann\n", " NaN\n", " NaN\n", " error\n", @@ -295,8 +301,8 @@ " \n", " \n", " 0\n", - " pinecone-ood\n", - " 76.87\n", + " scann\n", + " 107.41\n", " 0.91\n", " ok\n", " 1\n", @@ -305,8 +311,8 @@ " \n", " \n", " 1\n", - " zilliz\n", - " 73.51\n", + " pinecone-ood\n", + " 76.87\n", " 0.91\n", " ok\n", " 2\n", @@ -315,9 +321,9 @@ " \n", " \n", " 2\n", - " pyanns\n", - " 55.50\n", - " 0.92\n", + " zilliz\n", + " 73.51\n", + " 0.91\n", " ok\n", " 3\n", " ood\n", @@ -325,8 +331,8 @@ " \n", " \n", " 3\n", - " scann\n", - " 32.26\n", + " pyanns\n", + " 55.50\n", " 0.92\n", " ok\n", " 4\n", @@ -406,7 +412,7 @@ " \n", " 11\n", " diskann\n", - " 6.41\n", + " 6.30\n", " 0.91\n", " ok\n", " 12\n", @@ -505,7 +511,7 @@ " \n", " \n", " 7\n", - " sustech-whu\n", + " spmat\n", " NaN\n", " NaN\n", " error\n", @@ -515,7 +521,7 @@ " \n", " \n", " 8\n", - " spmat\n", + " sustech-whu\n", " NaN\n", " NaN\n", " error\n", @@ -538,13 +544,13 @@ "6 faissplus 8.49 0.91 ok 7 filter \n", "7 faiss 7.33 0.99 ok 8 filter \n", "8 cufe 6.32 0.94 ok 9 filter \n", - "9 fdufilterdiskann NaN NaN error 10 filter \n", - "10 dhq NaN NaN error 11 filter \n", + "9 dhq NaN NaN error 10 filter \n", + "10 fdufilterdiskann NaN NaN error 11 filter \n", "11 hwtl_sdu_anns_filter NaN NaN error 12 filter \n", - "0 pinecone-ood 76.87 0.91 ok 1 ood \n", - "1 zilliz 73.51 0.91 ok 2 ood \n", - "2 pyanns 55.50 0.92 ok 3 ood \n", - "3 scann 32.26 0.92 ok 4 ood \n", + "0 scann 107.41 0.91 ok 1 ood \n", + "1 pinecone-ood 76.87 0.91 ok 2 ood \n", + "2 zilliz 73.51 0.91 ok 3 ood \n", + "3 pyanns 55.50 0.92 ok 4 ood \n", "4 sustech-ood 28.46 0.92 ok 5 ood \n", "5 mysteryann-dif 27.95 0.91 ok 6 ood \n", "6 mysteryann 26.56 0.91 ok 7 ood \n", @@ -552,7 +558,7 @@ "8 puck 18.96 0.91 ok 9 ood \n", "9 ngt 11.92 0.95 ok 10 ood \n", "10 epsearch 7.66 0.92 ok 11 ood \n", - "11 diskann 6.41 0.91 ok 12 ood \n", + "11 diskann 6.30 0.91 ok 12 ood \n", "12 cufe 5.40 0.91 ok 13 ood \n", "13 puck-fizz NaN NaN error 14 ood \n", "0 zilliz 34.76 0.93 ok 1 sparse \n", @@ -562,8 +568,8 @@ "4 nle 2.95 0.94 ok 5 sparse \n", "5 cufe 0.08 0.99 ok 6 sparse \n", "6 linscan 0.06 0.98 ok 7 sparse \n", - "7 sustech-whu NaN NaN error 8 sparse \n", - "8 spmat NaN NaN error 9 sparse \n", + "7 spmat NaN NaN error 8 sparse \n", + "8 sustech-whu NaN NaN error 9 sparse \n", "\n", " dataset \n", "0 yfcc-10M \n", @@ -603,7 +609,7 @@ "8 sparse-full " ] }, - "execution_count": 7, + "execution_count": 38, "metadata": {}, "output_type": "execute_result" } @@ -616,7 +622,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 39, "id": "fbc47e1b-a441-413a-8166-bb834a386cc3", "metadata": {}, "outputs": [ @@ -723,7 +729,7 @@ " \n", " \n", " 7\n", - " sustech-whu\n", + " spmat\n", " NaN\n", " NaN\n", " error\n", @@ -733,7 +739,7 @@ " \n", " \n", " 8\n", - " spmat\n", + " sustech-whu\n", " NaN\n", " NaN\n", " error\n", @@ -743,8 +749,8 @@ " \n", " \n", " 0\n", - " pinecone-ood\n", - " 76.87\n", + " scann\n", + " 107.41\n", " 0.91\n", " ok\n", " 1\n", @@ -753,8 +759,8 @@ " \n", " \n", " 1\n", - " zilliz\n", - " 73.51\n", + " pinecone-ood\n", + " 76.87\n", " 0.91\n", " ok\n", " 2\n", @@ -763,9 +769,9 @@ " \n", " \n", " 2\n", - " pyanns\n", - " 55.50\n", - " 0.92\n", + " zilliz\n", + " 73.51\n", + " 0.91\n", " ok\n", " 3\n", " ood\n", @@ -773,8 +779,8 @@ " \n", " \n", " 3\n", - " scann\n", - " 32.26\n", + " pyanns\n", + " 55.50\n", " 0.92\n", " ok\n", " 4\n", @@ -854,7 +860,7 @@ " \n", " 11\n", " diskann\n", - " 6.41\n", + " 6.30\n", " 0.91\n", " ok\n", " 12\n", @@ -973,7 +979,7 @@ " \n", " \n", " 9\n", - " fdufilterdiskann\n", + " dhq\n", " NaN\n", " NaN\n", " error\n", @@ -983,7 +989,7 @@ " \n", " \n", " 10\n", - " dhq\n", + " fdufilterdiskann\n", " NaN\n", " NaN\n", " error\n", @@ -1014,12 +1020,12 @@ "4 nle 2.95 0.94 ok 5 sparse \n", "5 cufe 0.08 0.99 ok 6 sparse \n", "6 linscan 0.06 0.98 ok 7 sparse \n", - "7 sustech-whu NaN NaN error 8 sparse \n", - "8 spmat NaN NaN error 9 sparse \n", - "0 pinecone-ood 76.87 0.91 ok 1 ood \n", - "1 zilliz 73.51 0.91 ok 2 ood \n", - "2 pyanns 55.50 0.92 ok 3 ood \n", - "3 scann 32.26 0.92 ok 4 ood \n", + "7 spmat NaN NaN error 8 sparse \n", + "8 sustech-whu NaN NaN error 9 sparse \n", + "0 scann 107.41 0.91 ok 1 ood \n", + "1 pinecone-ood 76.87 0.91 ok 2 ood \n", + "2 zilliz 73.51 0.91 ok 3 ood \n", + "3 pyanns 55.50 0.92 ok 4 ood \n", "4 sustech-ood 28.46 0.92 ok 5 ood \n", "5 mysteryann-dif 27.95 0.91 ok 6 ood \n", "6 mysteryann 26.56 0.91 ok 7 ood \n", @@ -1027,7 +1033,7 @@ "8 puck 18.96 0.91 ok 9 ood \n", "9 ngt 11.92 0.95 ok 10 ood \n", "10 epsearch 7.66 0.92 ok 11 ood \n", - "11 diskann 6.41 0.91 ok 12 ood \n", + "11 diskann 6.30 0.91 ok 12 ood \n", "12 cufe 5.40 0.91 ok 13 ood \n", "13 puck-fizz NaN NaN error 14 ood \n", "0 zilliz 213.29 0.93 ok 1 filter \n", @@ -1039,8 +1045,8 @@ "6 faissplus 8.49 0.91 ok 7 filter \n", "7 faiss 7.33 0.99 ok 8 filter \n", "8 cufe 6.32 0.94 ok 9 filter \n", - "9 fdufilterdiskann NaN NaN error 10 filter \n", - "10 dhq NaN NaN error 11 filter \n", + "9 dhq NaN NaN error 10 filter \n", + "10 fdufilterdiskann NaN NaN error 11 filter \n", "11 hwtl_sdu_anns_filter NaN NaN error 12 filter \n", "\n", " dataset \n", @@ -1081,7 +1087,7 @@ "11 yfcc-10M " ] }, - "execution_count": 8, + "execution_count": 39, "metadata": {}, "output_type": "execute_result" } @@ -1095,7 +1101,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 40, "id": "c90fba89-728a-4f54-bb80-95834bd33005", "metadata": {}, "outputs": [], @@ -1111,7 +1117,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 41, "id": "f4e617b4-112b-45e1-bc3b-2692d980d19f", "metadata": {}, "outputs": [], @@ -1126,7 +1132,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 42, "id": "8d2653b4-f070-4761-8b5e-9a9fa03f19cc", "metadata": {}, "outputs": [], @@ -1139,7 +1145,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 43, "id": "d0accb53-c34b-418e-85a4-253c46cdb90b", "metadata": {}, "outputs": [], @@ -1162,7 +1168,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 44, "id": "5cee1395-0e41-4ba4-b586-f9c4b56a9024", "metadata": {}, "outputs": [ @@ -1171,25 +1177,25 @@ "text/html": [ "\n", - "\n", + "
\n", " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", " \n", @@ -1206,181 +1212,181 @@ " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", "
trackfiltersparseoodfiltersparseood
 rankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqps
rank
11zilliz213.2850591zilliz34.7599761pinecone-ood76.865423
22pinecone146.7212422pyanns26.8585192zilliz73.509610
33puck62.2576953pinecone_smips12.0031823pyanns55.504352
44parlayivf55.0331984shnsw8.2472774scann32.256046
55wm_filter20.8748575nle2.9456265sustech-ood28.458263
66pyanns8.9880226cufe0.0849766mysteryann-dif27.946097
77faissplus8.4932647linscan0.0607567mysteryann26.560867
88faiss7.327855sustech-whunan8vamana19.965517
99cufe6.318462spmatnan9puck18.960224
10fdufilterdiskannnannannan10ngt11.921399
11dhqnannannan11epsearch7.663219
12hwtl_sdu_anns_filternannannan12diskann6.406821
13nannannannan13cufe5.404492
14nannannannanpuck-fizznan11zilliz213.2850591zilliz34.7599761scann107.413009
22pinecone146.7212422pyanns26.8585192pinecone-ood76.865423
33puck62.2576953pinecone_smips12.0031823zilliz73.509610
44parlayivf55.0331984shnsw8.2472774pyanns55.504352
55wm_filter20.8748575nle2.9456265sustech-ood28.458263
66pyanns8.9880226cufe0.0849766mysteryann-dif27.946097
77faissplus8.4932647linscan0.0607567mysteryann26.560867
88faiss7.327855spmatnan8vamana19.965517
99cufe6.318462sustech-whunan9puck18.960224
10dhqnannannan10ngt11.921399
11fdufilterdiskannnannannan11epsearch7.663219
12hwtl_sdu_anns_filternannannan12diskann6.295839
13nannannannan13cufe5.404492
14nannannannanpuck-fizznan
\n" ], "text/plain": [ - "" + "" ] }, - "execution_count": 13, + "execution_count": 44, "metadata": {}, "output_type": "execute_result" } @@ -1394,7 +1400,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 45, "id": "884f5113-01a9-42a7-9262-5cea8e9cd751", "metadata": {}, "outputs": [ @@ -1402,10 +1408,10 @@ "name": "stdout", "output_type": "stream", "text": [ - "WARNING: Using error file path for sparse/sustech-whu\n", "WARNING: Using error file path for sparse/spmat\n", - "WARNING: Using error file path for filter/fdufilterdiskann\n", + "WARNING: Using error file path for sparse/sustech-whu\n", "WARNING: Using error file path for filter/dhq\n", + "WARNING: Using error file path for filter/fdufilterdiskann\n", "WARNING: Using error file path for filter/hwtl_sdu_anns_filter\n", "WARNING: Using error file path for ood/puck-fizz\n" ] @@ -1413,179 +1419,179 @@ { "data": { "text/html": [ - "\n", + "
\n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", "
filtersparseoodfiltersparseood
rankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqps
1
zilliz
 213.3K1
zilliz
 34.8K1
pinecone-ood
 76.9K
2
pinecone
 146.7K2
pyanns
 26.9K2
zilliz
 73.5K
3
puck
 62.3K3
pinecone_smips
 12.0K3
pyanns
 55.5K
4
parlayivf
 55.0K4
shnsw
 8.2K4
scann
 32.3K
5
wm_filter
 20.9K5
nle
 2.9K5
sustech-ood
 28.5K
6
pyanns
 9.0K6
cufe
 0.1K6
mysteryann-dif
 27.9K
7
faissplus
 8.5K7
linscan
 0.1K7
mysteryann
 26.6K
8
faiss
 7.3K
sustech-whu
8
vamana
 20.0K
9
cufe
 6.3K
spmat
9
puck
 19.0K
fdufilterdiskann
10
ngt
 11.9K
dhq
11
epsearch
 7.7K
hwtl_sdu_anns_filter
12
diskann
 6.4K
13
cufe
 5.4K
puck-fizz
1
zilliz
 213.3K1
zilliz
 34.8K1
scann
 107.4K
2
pinecone
 146.7K2
pyanns
 26.9K2
pinecone-ood
 76.9K
3
puck
 62.3K3
pinecone_smips
 12.0K3
zilliz
 73.5K
4
parlayivf
 55.0K4
shnsw
 8.2K4
pyanns
 55.5K
5
wm_filter
 20.9K5
nle
 2.9K5
sustech-ood
 28.5K
6
pyanns
 9.0K6
cufe
 0.1K6
mysteryann-dif
 27.9K
7
faissplus
 8.5K7
linscan
 0.1K7
mysteryann
 26.6K
8
faiss
 7.3K
spmat
8
vamana
 20.0K
9
cufe
 6.3K
sustech-whu
9
puck
 19.0K
dhq
10
ngt
 11.9K
fdufilterdiskann
11
epsearch
 7.7K
hwtl_sdu_anns_filter
12
diskann
 6.3K
13
cufe
 5.4K
puck-fizz
\n" @@ -1594,7 +1600,7 @@ "" ] }, - "execution_count": 14, + "execution_count": 45, "metadata": {}, "output_type": "execute_result" } @@ -1622,7 +1628,7 @@ "new_html = new_html.replace(\"nan\",\"\")\n", "\n", "# insert links for specific use cases (ie, errors)\n", - "new_html = helper.replace_table_with_links(new_html, swap_df)\n", + "new_html = helper.replace_table_with_links(new_html, swap_df, IGNORE_ALGOS)\n", "\n", "# display in-line\n", "HTML(new_html)" @@ -1630,7 +1636,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 46, "id": "25fc371b-1a14-4fca-97de-acd79070c0ad", "metadata": {}, "outputs": [ diff --git a/neurips23/latitude/helper.py b/neurips23/latitude/helper.py index e46af237..629a765c 100644 --- a/neurips23/latitude/helper.py +++ b/neurips23/latitude/helper.py @@ -19,7 +19,7 @@ def remove_style_prefix(html): else: return html -def replace_table_with_links(html, df): +def replace_table_with_links(html, df, ignore_algos=[]): '''This is a messy function but seems to work well. It replaces table cells with appropriate links by iterating the rows and cols of table and locating the cell html via regex. It may be better to use BeautifulSoup and related scraping/html helper library.''' @@ -36,6 +36,9 @@ def replace_table_with_links(html, df): track = col[0] algo = matches.groups()[1] if algo=="": continue + if algo in ignore_algos: + print("WARNING: Ignoring %s" % algo) + continue link_path = os.path.join( CDIR, "%s__%s.sh" % (track, algo) ) if not os.path.exists(link_path): raise Exception("Cannot locate run path" + link_path)