From 7f5b1cc5aafe2a0ba25cd0d7259c89349d93b0d7 Mon Sep 17 00:00:00 2001 From: George Williams Date: Fri, 13 Sep 2024 13:23:10 -0700 Subject: [PATCH 1/3] adding algo ignore for hanns temporarily; rerender markdown for new ranking --- neurips23/latitude/analysis.ipynb | 705 ++++++++++++------------------ neurips23/latitude/helper.py | 5 +- 2 files changed, 288 insertions(+), 422 deletions(-) diff --git a/neurips23/latitude/analysis.ipynb b/neurips23/latitude/analysis.ipynb index 524e519c..01f9fd94 100644 --- a/neurips23/latitude/analysis.ipynb +++ b/neurips23/latitude/analysis.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": 15, "id": "21ac7846-3ebf-4f2a-b545-be471d0ce5a8", "metadata": {}, "outputs": [], @@ -19,12 +19,15 @@ "RECALL_AP_THRESHOLD = 0.9\n", "\n", "# Markdown template file\n", - "MARKDOWN = \"./_latitude-m4-metal-medium.md\"" + "MARKDOWN = \"./_latitude-m4-metal-medium.md\"\n", + "\n", + "# Algorithms to ignore - mostly to deal with transient issues like algorithm not yet merged\n", + "IGNORE_ALGOS=[\"ood__hanns\"] " ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 16, "id": "936c9c86-f611-4b3f-ae6b-d9d02347a4e2", "metadata": {}, "outputs": [], @@ -265,7 +268,7 @@ " \n", " \n", " 9\n", - " fdufilterdiskann\n", + " dhq\n", " NaN\n", " NaN\n", " error\n", @@ -275,7 +278,7 @@ " \n", " \n", " 10\n", - " dhq\n", + " hwtl_sdu_anns_filter\n", " NaN\n", " NaN\n", " error\n", @@ -285,7 +288,7 @@ " \n", " \n", " 11\n", - " hwtl_sdu_anns_filter\n", + " fdufilterdiskann\n", " NaN\n", " NaN\n", " error\n", @@ -295,8 +298,8 @@ " \n", " \n", " 0\n", - " pinecone-ood\n", - " 76.87\n", + " scann\n", + " 107.41\n", " 0.91\n", " ok\n", " 1\n", @@ -305,8 +308,8 @@ " \n", " \n", " 1\n", - " zilliz\n", - " 73.51\n", + " pinecone-ood\n", + " 76.87\n", " 0.91\n", " ok\n", " 2\n", @@ -315,9 +318,9 @@ " \n", " \n", " 2\n", - " pyanns\n", - " 55.50\n", - " 0.92\n", + " zilliz\n", + " 73.51\n", + " 0.91\n", " ok\n", " 3\n", " ood\n", @@ -325,8 +328,8 @@ " \n", " \n", " 3\n", - " scann\n", - " 32.26\n", + " pyanns\n", + " 55.50\n", " 0.92\n", " ok\n", " 4\n", @@ -406,7 +409,7 @@ " \n", " 11\n", " diskann\n", - " 6.41\n", + " 6.30\n", " 0.91\n", " ok\n", " 12\n", @@ -434,6 +437,16 @@ " text2image-10M\n", " \n", " \n", + " 14\n", + " hanns\n", + " NaN\n", + " NaN\n", + " error\n", + " 15\n", + " ood\n", + " text2image-10M\n", + " \n", + " \n", " 0\n", " zilliz\n", " 34.76\n", @@ -505,7 +518,7 @@ " \n", " \n", " 7\n", - " sustech-whu\n", + " spmat\n", " NaN\n", " NaN\n", " error\n", @@ -515,7 +528,7 @@ " \n", " \n", " 8\n", - " spmat\n", + " sustech-whu\n", " NaN\n", " NaN\n", " error\n", @@ -538,13 +551,13 @@ "6 faissplus 8.49 0.91 ok 7 filter \n", "7 faiss 7.33 0.99 ok 8 filter \n", "8 cufe 6.32 0.94 ok 9 filter \n", - "9 fdufilterdiskann NaN NaN error 10 filter \n", - "10 dhq NaN NaN error 11 filter \n", - "11 hwtl_sdu_anns_filter NaN NaN error 12 filter \n", - "0 pinecone-ood 76.87 0.91 ok 1 ood \n", - "1 zilliz 73.51 0.91 ok 2 ood \n", - "2 pyanns 55.50 0.92 ok 3 ood \n", - "3 scann 32.26 0.92 ok 4 ood \n", + "9 dhq NaN NaN error 10 filter \n", + "10 hwtl_sdu_anns_filter NaN NaN error 11 filter \n", + "11 fdufilterdiskann NaN NaN error 12 filter \n", + "0 scann 107.41 0.91 ok 1 ood \n", + "1 pinecone-ood 76.87 0.91 ok 2 ood \n", + "2 zilliz 73.51 0.91 ok 3 ood \n", + "3 pyanns 55.50 0.92 ok 4 ood \n", "4 sustech-ood 28.46 0.92 ok 5 ood \n", "5 mysteryann-dif 27.95 0.91 ok 6 ood \n", "6 mysteryann 26.56 0.91 ok 7 ood \n", @@ -552,9 +565,10 @@ "8 puck 18.96 0.91 ok 9 ood \n", "9 ngt 11.92 0.95 ok 10 ood \n", "10 epsearch 7.66 0.92 ok 11 ood \n", - "11 diskann 6.41 0.91 ok 12 ood \n", + "11 diskann 6.30 0.91 ok 12 ood \n", "12 cufe 5.40 0.91 ok 13 ood \n", "13 puck-fizz NaN NaN error 14 ood \n", + "14 hanns NaN NaN error 15 ood \n", "0 zilliz 34.76 0.93 ok 1 sparse \n", "1 pyanns 26.86 0.92 ok 2 sparse \n", "2 pinecone_smips 12.00 0.90 ok 3 sparse \n", @@ -562,8 +576,8 @@ "4 nle 2.95 0.94 ok 5 sparse \n", "5 cufe 0.08 0.99 ok 6 sparse \n", "6 linscan 0.06 0.98 ok 7 sparse \n", - "7 sustech-whu NaN NaN error 8 sparse \n", - "8 spmat NaN NaN error 9 sparse \n", + "7 spmat NaN NaN error 8 sparse \n", + "8 sustech-whu NaN NaN error 9 sparse \n", "\n", " dataset \n", "0 yfcc-10M \n", @@ -592,6 +606,7 @@ "11 text2image-10M \n", "12 text2image-10M \n", "13 text2image-10M \n", + "14 text2image-10M \n", "0 sparse-full \n", "1 sparse-full \n", "2 sparse-full \n", @@ -723,7 +738,7 @@ " \n", " \n", " 7\n", - " sustech-whu\n", + " spmat\n", " NaN\n", " NaN\n", " error\n", @@ -733,7 +748,7 @@ " \n", " \n", " 8\n", - " spmat\n", + " sustech-whu\n", " NaN\n", " NaN\n", " error\n", @@ -743,8 +758,8 @@ " \n", " \n", " 0\n", - " pinecone-ood\n", - " 76.87\n", + " scann\n", + " 107.41\n", " 0.91\n", " ok\n", " 1\n", @@ -753,8 +768,8 @@ " \n", " \n", " 1\n", - " zilliz\n", - " 73.51\n", + " pinecone-ood\n", + " 76.87\n", " 0.91\n", " ok\n", " 2\n", @@ -763,9 +778,9 @@ " \n", " \n", " 2\n", - " pyanns\n", - " 55.50\n", - " 0.92\n", + " zilliz\n", + " 73.51\n", + " 0.91\n", " ok\n", " 3\n", " ood\n", @@ -773,8 +788,8 @@ " \n", " \n", " 3\n", - " scann\n", - " 32.26\n", + " pyanns\n", + " 55.50\n", " 0.92\n", " ok\n", " 4\n", @@ -854,7 +869,7 @@ " \n", " 11\n", " diskann\n", - " 6.41\n", + " 6.30\n", " 0.91\n", " ok\n", " 12\n", @@ -882,6 +897,16 @@ " text2image-10M\n", " \n", " \n", + " 14\n", + " hanns\n", + " NaN\n", + " NaN\n", + " error\n", + " 15\n", + " ood\n", + " text2image-10M\n", + " \n", + " \n", " 0\n", " zilliz\n", " 213.29\n", @@ -973,7 +998,7 @@ " \n", " \n", " 9\n", - " fdufilterdiskann\n", + " dhq\n", " NaN\n", " NaN\n", " error\n", @@ -983,7 +1008,7 @@ " \n", " \n", " 10\n", - " dhq\n", + " hwtl_sdu_anns_filter\n", " NaN\n", " NaN\n", " error\n", @@ -993,7 +1018,7 @@ " \n", " \n", " 11\n", - " hwtl_sdu_anns_filter\n", + " fdufilterdiskann\n", " NaN\n", " NaN\n", " error\n", @@ -1014,12 +1039,12 @@ "4 nle 2.95 0.94 ok 5 sparse \n", "5 cufe 0.08 0.99 ok 6 sparse \n", "6 linscan 0.06 0.98 ok 7 sparse \n", - "7 sustech-whu NaN NaN error 8 sparse \n", - "8 spmat NaN NaN error 9 sparse \n", - "0 pinecone-ood 76.87 0.91 ok 1 ood \n", - "1 zilliz 73.51 0.91 ok 2 ood \n", - "2 pyanns 55.50 0.92 ok 3 ood \n", - "3 scann 32.26 0.92 ok 4 ood \n", + "7 spmat NaN NaN error 8 sparse \n", + "8 sustech-whu NaN NaN error 9 sparse \n", + "0 scann 107.41 0.91 ok 1 ood \n", + "1 pinecone-ood 76.87 0.91 ok 2 ood \n", + "2 zilliz 73.51 0.91 ok 3 ood \n", + "3 pyanns 55.50 0.92 ok 4 ood \n", "4 sustech-ood 28.46 0.92 ok 5 ood \n", "5 mysteryann-dif 27.95 0.91 ok 6 ood \n", "6 mysteryann 26.56 0.91 ok 7 ood \n", @@ -1027,9 +1052,10 @@ "8 puck 18.96 0.91 ok 9 ood \n", "9 ngt 11.92 0.95 ok 10 ood \n", "10 epsearch 7.66 0.92 ok 11 ood \n", - "11 diskann 6.41 0.91 ok 12 ood \n", + "11 diskann 6.30 0.91 ok 12 ood \n", "12 cufe 5.40 0.91 ok 13 ood \n", "13 puck-fizz NaN NaN error 14 ood \n", + "14 hanns NaN NaN error 15 ood \n", "0 zilliz 213.29 0.93 ok 1 filter \n", "1 pinecone 146.72 0.92 ok 2 filter \n", "2 puck 62.26 0.94 ok 3 filter \n", @@ -1039,9 +1065,9 @@ "6 faissplus 8.49 0.91 ok 7 filter \n", "7 faiss 7.33 0.99 ok 8 filter \n", "8 cufe 6.32 0.94 ok 9 filter \n", - "9 fdufilterdiskann NaN NaN error 10 filter \n", - "10 dhq NaN NaN error 11 filter \n", - "11 hwtl_sdu_anns_filter NaN NaN error 12 filter \n", + "9 dhq NaN NaN error 10 filter \n", + "10 hwtl_sdu_anns_filter NaN NaN error 11 filter \n", + "11 fdufilterdiskann NaN NaN error 12 filter \n", "\n", " dataset \n", "0 sparse-full \n", @@ -1067,6 +1093,7 @@ "11 text2image-10M \n", "12 text2image-10M \n", "13 text2image-10M \n", + "14 text2image-10M \n", "0 yfcc-10M \n", "1 yfcc-10M \n", "2 yfcc-10M \n", @@ -1171,25 +1198,25 @@ "text/html": [ "\n", - "\n", + "
\n", " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", " \n", @@ -1206,178 +1233,190 @@ " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", "
trackfiltersparseoodfiltersparseood
 rankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqps
rank
11zilliz213.2850591zilliz34.7599761pinecone-ood76.865423
22pinecone146.7212422pyanns26.8585192zilliz73.509610
33puck62.2576953pinecone_smips12.0031823pyanns55.504352
44parlayivf55.0331984shnsw8.2472774scann32.256046
55wm_filter20.8748575nle2.9456265sustech-ood28.458263
66pyanns8.9880226cufe0.0849766mysteryann-dif27.946097
77faissplus8.4932647linscan0.0607567mysteryann26.560867
88faiss7.327855sustech-whunan8vamana19.965517
99cufe6.318462spmatnan9puck18.960224
10fdufilterdiskannnannannan10ngt11.921399
11dhqnannannan11epsearch7.663219
12hwtl_sdu_anns_filternannannan12diskann6.406821
13nannannannan13cufe5.404492
14nannannannanpuck-fizznan11zilliz213.2850591zilliz34.7599761scann107.413009
22pinecone146.7212422pyanns26.8585192pinecone-ood76.865423
33puck62.2576953pinecone_smips12.0031823zilliz73.509610
44parlayivf55.0331984shnsw8.2472774pyanns55.504352
55wm_filter20.8748575nle2.9456265sustech-ood28.458263
66pyanns8.9880226cufe0.0849766mysteryann-dif27.946097
77faissplus8.4932647linscan0.0607567mysteryann26.560867
88faiss7.327855spmatnan8vamana19.965517
99cufe6.318462sustech-whunan9puck18.960224
10dhqnannannan10ngt11.921399
11hwtl_sdu_anns_filternannannan11epsearch7.663219
12fdufilterdiskannnannannan12diskann6.295839
13nannannannan13cufe5.404492
14nannannannanpuck-fizznan
15nannannannanhannsnan
\n" ], "text/plain": [ - "" + "" ] }, "execution_count": 13, @@ -1402,201 +1441,25 @@ "name": "stdout", "output_type": "stream", "text": [ - "WARNING: Using error file path for sparse/sustech-whu\n", "WARNING: Using error file path for sparse/spmat\n", - "WARNING: Using error file path for filter/fdufilterdiskann\n", + "WARNING: Using error file path for sparse/sustech-whu\n", "WARNING: Using error file path for filter/dhq\n", "WARNING: Using error file path for filter/hwtl_sdu_anns_filter\n", + "WARNING: Using error file path for filter/fdufilterdiskann\n", "WARNING: Using error file path for ood/puck-fizz\n" ] }, { - "data": { - "text/html": [ - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
filtersparseood
rankalgorithmqpsrankalgorithmqpsrankalgorithmqps
1
zilliz
 213.3K1
zilliz
 34.8K1
pinecone-ood
 76.9K
2
pinecone
 146.7K2
pyanns
 26.9K2
zilliz
 73.5K
3
puck
 62.3K3
pinecone_smips
 12.0K3
pyanns
 55.5K
4
parlayivf
 55.0K4
shnsw
 8.2K4
scann
 32.3K
5
wm_filter
 20.9K5
nle
 2.9K5
sustech-ood
 28.5K
6
pyanns
 9.0K6
cufe
 0.1K6
mysteryann-dif
 27.9K
7
faissplus
 8.5K7
linscan
 0.1K7
mysteryann
 26.6K
8
faiss
 7.3K
sustech-whu
8
vamana
 20.0K
9
cufe
 6.3K
spmat
9
puck
 19.0K
fdufilterdiskann
10
ngt
 11.9K
dhq
11
epsearch
 7.7K
hwtl_sdu_anns_filter
12
diskann
 6.4K
13
cufe
 5.4K
puck-fizz
\n" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" + "ename": "Exception", + "evalue": "Cannot locate run pathcommands/ood__hanns.sh", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mException\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[14], line 23\u001b[0m\n\u001b[1;32m 20\u001b[0m new_html \u001b[38;5;241m=\u001b[39m new_html\u001b[38;5;241m.\u001b[39mreplace(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnan\u001b[39m\u001b[38;5;124m\"\u001b[39m,\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 22\u001b[0m \u001b[38;5;66;03m# insert links for specific use cases (ie, errors)\u001b[39;00m\n\u001b[0;32m---> 23\u001b[0m new_html \u001b[38;5;241m=\u001b[39m \u001b[43mhelper\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mreplace_table_with_links\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnew_html\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mswap_df\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 25\u001b[0m \u001b[38;5;66;03m# display in-line\u001b[39;00m\n\u001b[1;32m 26\u001b[0m HTML(new_html)\n", + "File \u001b[0;32m~/Projects/BigANN/harsha/big-ann-benchmarks/neurips23/latitude/helper.py:41\u001b[0m, in \u001b[0;36mreplace_table_with_links\u001b[0;34m(html, df)\u001b[0m\n\u001b[1;32m 39\u001b[0m link_path \u001b[38;5;241m=\u001b[39m os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mjoin( CDIR, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m__\u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m.sh\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m%\u001b[39m (track, algo) )\n\u001b[1;32m 40\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mexists(link_path):\n\u001b[0;32m---> 41\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mCannot locate run path\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m+\u001b[39m link_path)\n\u001b[1;32m 42\u001b[0m err_path \u001b[38;5;241m=\u001b[39m os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mjoin( EDIR, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m__\u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m.txt\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m%\u001b[39m (track, algo) )\n\u001b[1;32m 43\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mexists(err_path):\n", + "\u001b[0;31mException\u001b[0m: Cannot locate run pathcommands/ood__hanns.sh" + ] } ], "source": [ diff --git a/neurips23/latitude/helper.py b/neurips23/latitude/helper.py index e46af237..629a765c 100644 --- a/neurips23/latitude/helper.py +++ b/neurips23/latitude/helper.py @@ -19,7 +19,7 @@ def remove_style_prefix(html): else: return html -def replace_table_with_links(html, df): +def replace_table_with_links(html, df, ignore_algos=[]): '''This is a messy function but seems to work well. It replaces table cells with appropriate links by iterating the rows and cols of table and locating the cell html via regex. It may be better to use BeautifulSoup and related scraping/html helper library.''' @@ -36,6 +36,9 @@ def replace_table_with_links(html, df): track = col[0] algo = matches.groups()[1] if algo=="": continue + if algo in ignore_algos: + print("WARNING: Ignoring %s" % algo) + continue link_path = os.path.join( CDIR, "%s__%s.sh" % (track, algo) ) if not os.path.exists(link_path): raise Exception("Cannot locate run path" + link_path) From e5125a69f90a521c3628833bfeff0cfc142fb741 Mon Sep 17 00:00:00 2001 From: George Williams Date: Fri, 13 Sep 2024 13:24:08 -0700 Subject: [PATCH 2/3] run all nb cells; rerender markdown and rankings --- neurips23/latitude-m4-metal-medium.md | 289 +++++------ neurips23/latitude/analysis.ipynb | 664 +++++++++++++++++--------- 2 files changed, 576 insertions(+), 377 deletions(-) diff --git a/neurips23/latitude-m4-metal-medium.md b/neurips23/latitude-m4-metal-medium.md index 031e2a07..864cf3cf 100644 --- a/neurips23/latitude-m4-metal-medium.md +++ b/neurips23/latitude-m4-metal-medium.md @@ -1,179 +1,190 @@ # Eval On AMD 3GHz/16-Core + 125GB RAM + NVMe SSD (Bare Metal) - +
- - - + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + + + + + + + + + + + +
filtersparseoodfiltersparseood
rankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqps
1
zilliz
 213.3K1
zilliz
 34.8K1
pinecone-ood
 76.9K1
zilliz
 213.3K1
zilliz
 34.8K1
scann
 107.4K
2
pinecone
 146.7K2
pyanns
 26.9K2
zilliz
 73.5K2
pinecone
 146.7K2
pyanns
 26.9K2
pinecone-ood
 76.9K
3
puck
 62.3K3
pinecone_smips
 12.0K3
pyanns
 55.5K3
puck
 62.3K3
pinecone_smips
 12.0K3
zilliz
 73.5K
4
parlayivf
 55.0K4
shnsw
 8.2K4
scann
 32.3K4
parlayivf
 55.0K4
shnsw
 8.2K4
pyanns
 55.5K
5
wm_filter
 20.9K5
nle
 2.9K5
sustech-ood
 28.5K5
wm_filter
 20.9K5
nle
 2.9K5
sustech-ood
 28.5K
6
pyanns
 9.0K6
cufe
 0.1K6
mysteryann-dif
 27.9K6
pyanns
 9.0K6
cufe
 0.1K6
mysteryann-dif
 27.9K
7
faissplus
 8.5K7
linscan
 0.1K7
mysteryann
 26.6K7
faissplus
 8.5K7
linscan
 0.1K7
mysteryann
 26.6K
8
faiss
 7.3K
sustech-whu
8
vamana
 20.0K8
faiss
 7.3K
spmat
8
vamana
 20.0K
9
cufe
 6.3K
spmat
9
puck
 19.0K9
cufe
 6.3K
sustech-whu
9
puck
 19.0K
dhq
10
ngt
 11.9K
dhq
10
ngt
 11.9K
fdufilterdiskann
11
epsearch
 7.7K
fdufilterdiskann
11
epsearch
 7.7K
hwtl_sdu_anns_filter
12
diskann
 6.4K
hwtl_sdu_anns_filter
12
diskann
 6.3K
13
cufe
 5.4K13
cufe
 5.4K
puck-fizz
hanns
puck-fizz
diff --git a/neurips23/latitude/analysis.ipynb b/neurips23/latitude/analysis.ipynb index 01f9fd94..26ce6266 100644 --- a/neurips23/latitude/analysis.ipynb +++ b/neurips23/latitude/analysis.ipynb @@ -21,8 +21,8 @@ "# Markdown template file\n", "MARKDOWN = \"./_latitude-m4-metal-medium.md\"\n", "\n", - "# Algorithms to ignore - mostly to deal with transient issues like algorithm not yet merged\n", - "IGNORE_ALGOS=[\"ood__hanns\"] " + "# Algorithms to ignore - mostly to deal with transient issues like new algorithms not yet merged\n", + "IGNORE_ALGOS=[\"hanns\"] " ] }, { @@ -46,7 +46,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 17, "id": "071746d6-1cdc-404e-beef-d643596218fd", "metadata": {}, "outputs": [], @@ -58,7 +58,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 18, "id": "7193be1a-a6de-43df-9f60-9243f5ecf473", "metadata": {}, "outputs": [], @@ -70,7 +70,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 19, "id": "5c83a9db-fc3f-4813-91c8-a7a1275b086d", "metadata": {}, "outputs": [], @@ -82,7 +82,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 20, "id": "303b19d1-d0a6-4f0c-9dd0-08879df9cdd6", "metadata": {}, "outputs": [], @@ -141,7 +141,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 21, "id": "51cf0a75-1acf-44de-b7f1-2cf030cadd94", "metadata": {}, "outputs": [ @@ -278,7 +278,7 @@ " \n", " \n", " 10\n", - " hwtl_sdu_anns_filter\n", + " fdufilterdiskann\n", " NaN\n", " NaN\n", " error\n", @@ -288,7 +288,7 @@ " \n", " \n", " 11\n", - " fdufilterdiskann\n", + " hwtl_sdu_anns_filter\n", " NaN\n", " NaN\n", " error\n", @@ -428,7 +428,7 @@ " \n", " \n", " 13\n", - " puck-fizz\n", + " hanns\n", " NaN\n", " NaN\n", " error\n", @@ -438,7 +438,7 @@ " \n", " \n", " 14\n", - " hanns\n", + " puck-fizz\n", " NaN\n", " NaN\n", " error\n", @@ -552,8 +552,8 @@ "7 faiss 7.33 0.99 ok 8 filter \n", "8 cufe 6.32 0.94 ok 9 filter \n", "9 dhq NaN NaN error 10 filter \n", - "10 hwtl_sdu_anns_filter NaN NaN error 11 filter \n", - "11 fdufilterdiskann NaN NaN error 12 filter \n", + "10 fdufilterdiskann NaN NaN error 11 filter \n", + "11 hwtl_sdu_anns_filter NaN NaN error 12 filter \n", "0 scann 107.41 0.91 ok 1 ood \n", "1 pinecone-ood 76.87 0.91 ok 2 ood \n", "2 zilliz 73.51 0.91 ok 3 ood \n", @@ -567,8 +567,8 @@ "10 epsearch 7.66 0.92 ok 11 ood \n", "11 diskann 6.30 0.91 ok 12 ood \n", "12 cufe 5.40 0.91 ok 13 ood \n", - "13 puck-fizz NaN NaN error 14 ood \n", - "14 hanns NaN NaN error 15 ood \n", + "13 hanns NaN NaN error 14 ood \n", + "14 puck-fizz NaN NaN error 15 ood \n", "0 zilliz 34.76 0.93 ok 1 sparse \n", "1 pyanns 26.86 0.92 ok 2 sparse \n", "2 pinecone_smips 12.00 0.90 ok 3 sparse \n", @@ -618,7 +618,7 @@ "8 sparse-full " ] }, - "execution_count": 7, + "execution_count": 21, "metadata": {}, "output_type": "execute_result" } @@ -631,7 +631,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 22, "id": "fbc47e1b-a441-413a-8166-bb834a386cc3", "metadata": {}, "outputs": [ @@ -888,7 +888,7 @@ " \n", " \n", " 13\n", - " puck-fizz\n", + " hanns\n", " NaN\n", " NaN\n", " error\n", @@ -898,7 +898,7 @@ " \n", " \n", " 14\n", - " hanns\n", + " puck-fizz\n", " NaN\n", " NaN\n", " error\n", @@ -1008,7 +1008,7 @@ " \n", " \n", " 10\n", - " hwtl_sdu_anns_filter\n", + " fdufilterdiskann\n", " NaN\n", " NaN\n", " error\n", @@ -1018,7 +1018,7 @@ " \n", " \n", " 11\n", - " fdufilterdiskann\n", + " hwtl_sdu_anns_filter\n", " NaN\n", " NaN\n", " error\n", @@ -1054,8 +1054,8 @@ "10 epsearch 7.66 0.92 ok 11 ood \n", "11 diskann 6.30 0.91 ok 12 ood \n", "12 cufe 5.40 0.91 ok 13 ood \n", - "13 puck-fizz NaN NaN error 14 ood \n", - "14 hanns NaN NaN error 15 ood \n", + "13 hanns NaN NaN error 14 ood \n", + "14 puck-fizz NaN NaN error 15 ood \n", "0 zilliz 213.29 0.93 ok 1 filter \n", "1 pinecone 146.72 0.92 ok 2 filter \n", "2 puck 62.26 0.94 ok 3 filter \n", @@ -1066,8 +1066,8 @@ "7 faiss 7.33 0.99 ok 8 filter \n", "8 cufe 6.32 0.94 ok 9 filter \n", "9 dhq NaN NaN error 10 filter \n", - "10 hwtl_sdu_anns_filter NaN NaN error 11 filter \n", - "11 fdufilterdiskann NaN NaN error 12 filter \n", + "10 fdufilterdiskann NaN NaN error 11 filter \n", + "11 hwtl_sdu_anns_filter NaN NaN error 12 filter \n", "\n", " dataset \n", "0 sparse-full \n", @@ -1108,7 +1108,7 @@ "11 yfcc-10M " ] }, - "execution_count": 8, + "execution_count": 22, "metadata": {}, "output_type": "execute_result" } @@ -1122,7 +1122,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 23, "id": "c90fba89-728a-4f54-bb80-95834bd33005", "metadata": {}, "outputs": [], @@ -1138,7 +1138,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 24, "id": "f4e617b4-112b-45e1-bc3b-2692d980d19f", "metadata": {}, "outputs": [], @@ -1153,7 +1153,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 25, "id": "8d2653b4-f070-4761-8b5e-9a9fa03f19cc", "metadata": {}, "outputs": [], @@ -1166,7 +1166,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 26, "id": "d0accb53-c34b-418e-85a4-253c46cdb90b", "metadata": {}, "outputs": [], @@ -1189,7 +1189,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 27, "id": "5cee1395-0e41-4ba4-b586-f9c4b56a9024", "metadata": {}, "outputs": [ @@ -1198,25 +1198,25 @@ "text/html": [ "\n", - "\n", + "
\n", " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", " \n", @@ -1233,193 +1233,193 @@ " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", "
trackfiltersparseoodfiltersparseood
 rankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqps
rank
11zilliz213.2850591zilliz34.7599761scann107.413009
22pinecone146.7212422pyanns26.8585192pinecone-ood76.865423
33puck62.2576953pinecone_smips12.0031823zilliz73.509610
44parlayivf55.0331984shnsw8.2472774pyanns55.504352
55wm_filter20.8748575nle2.9456265sustech-ood28.458263
66pyanns8.9880226cufe0.0849766mysteryann-dif27.946097
77faissplus8.4932647linscan0.0607567mysteryann26.560867
88faiss7.327855spmatnan8vamana19.965517
99cufe6.318462sustech-whunan9puck18.960224
10dhqnannannan10ngt11.921399
11hwtl_sdu_anns_filternannannan11epsearch7.663219
12fdufilterdiskannnannannan12diskann6.295839
13nannannannan13cufe5.404492
14nannannannanpuck-fizznan
15nannannannanhannsnan11zilliz213.2850591zilliz34.7599761scann107.413009
22pinecone146.7212422pyanns26.8585192pinecone-ood76.865423
33puck62.2576953pinecone_smips12.0031823zilliz73.509610
44parlayivf55.0331984shnsw8.2472774pyanns55.504352
55wm_filter20.8748575nle2.9456265sustech-ood28.458263
66pyanns8.9880226cufe0.0849766mysteryann-dif27.946097
77faissplus8.4932647linscan0.0607567mysteryann26.560867
88faiss7.327855spmatnan8vamana19.965517
99cufe6.318462sustech-whunan9puck18.960224
10dhqnannannan10ngt11.921399
11fdufilterdiskannnannannan11epsearch7.663219
12hwtl_sdu_anns_filternannannan12diskann6.295839
13nannannannan13cufe5.404492
14nannannannanhannsnan
15nannannannanpuck-fizznan
\n" ], "text/plain": [ - "" + "" ] }, - "execution_count": 13, + "execution_count": 27, "metadata": {}, "output_type": "execute_result" } @@ -1433,7 +1433,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 28, "id": "884f5113-01a9-42a7-9262-5cea8e9cd751", "metadata": {}, "outputs": [ @@ -1444,22 +1444,210 @@ "WARNING: Using error file path for sparse/spmat\n", "WARNING: Using error file path for sparse/sustech-whu\n", "WARNING: Using error file path for filter/dhq\n", - "WARNING: Using error file path for filter/hwtl_sdu_anns_filter\n", "WARNING: Using error file path for filter/fdufilterdiskann\n", + "WARNING: Using error file path for filter/hwtl_sdu_anns_filter\n", + "WARNING: Ignoring hanns\n", "WARNING: Using error file path for ood/puck-fizz\n" ] }, { - "ename": "Exception", - "evalue": "Cannot locate run pathcommands/ood__hanns.sh", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mException\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[14], line 23\u001b[0m\n\u001b[1;32m 20\u001b[0m new_html \u001b[38;5;241m=\u001b[39m new_html\u001b[38;5;241m.\u001b[39mreplace(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnan\u001b[39m\u001b[38;5;124m\"\u001b[39m,\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 22\u001b[0m \u001b[38;5;66;03m# insert links for specific use cases (ie, errors)\u001b[39;00m\n\u001b[0;32m---> 23\u001b[0m new_html \u001b[38;5;241m=\u001b[39m \u001b[43mhelper\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mreplace_table_with_links\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnew_html\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mswap_df\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 25\u001b[0m \u001b[38;5;66;03m# display in-line\u001b[39;00m\n\u001b[1;32m 26\u001b[0m HTML(new_html)\n", - "File \u001b[0;32m~/Projects/BigANN/harsha/big-ann-benchmarks/neurips23/latitude/helper.py:41\u001b[0m, in \u001b[0;36mreplace_table_with_links\u001b[0;34m(html, df)\u001b[0m\n\u001b[1;32m 39\u001b[0m link_path \u001b[38;5;241m=\u001b[39m os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mjoin( CDIR, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m__\u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m.sh\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m%\u001b[39m (track, algo) )\n\u001b[1;32m 40\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mexists(link_path):\n\u001b[0;32m---> 41\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mCannot locate run path\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m+\u001b[39m link_path)\n\u001b[1;32m 42\u001b[0m err_path \u001b[38;5;241m=\u001b[39m os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mjoin( EDIR, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m__\u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m.txt\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m%\u001b[39m (track, algo) )\n\u001b[1;32m 43\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mexists(err_path):\n", - "\u001b[0;31mException\u001b[0m: Cannot locate run pathcommands/ood__hanns.sh" - ] + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
filtersparseood
rankalgorithmqpsrankalgorithmqpsrankalgorithmqps
1
zilliz
 213.3K1
zilliz
 34.8K1
scann
 107.4K
2
pinecone
 146.7K2
pyanns
 26.9K2
pinecone-ood
 76.9K
3
puck
 62.3K3
pinecone_smips
 12.0K3
zilliz
 73.5K
4
parlayivf
 55.0K4
shnsw
 8.2K4
pyanns
 55.5K
5
wm_filter
 20.9K5
nle
 2.9K5
sustech-ood
 28.5K
6
pyanns
 9.0K6
cufe
 0.1K6
mysteryann-dif
 27.9K
7
faissplus
 8.5K7
linscan
 0.1K7
mysteryann
 26.6K
8
faiss
 7.3K
spmat
8
vamana
 20.0K
9
cufe
 6.3K
sustech-whu
9
puck
 19.0K
dhq
10
ngt
 11.9K
fdufilterdiskann
11
epsearch
 7.7K
hwtl_sdu_anns_filter
12
diskann
 6.3K
13
cufe
 5.4K
hanns
puck-fizz
\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 28, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ @@ -1485,7 +1673,7 @@ "new_html = new_html.replace(\"nan\",\"\")\n", "\n", "# insert links for specific use cases (ie, errors)\n", - "new_html = helper.replace_table_with_links(new_html, swap_df)\n", + "new_html = helper.replace_table_with_links(new_html, swap_df, IGNORE_ALGOS)\n", "\n", "# display in-line\n", "HTML(new_html)" @@ -1493,7 +1681,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 29, "id": "25fc371b-1a14-4fca-97de-acd79070c0ad", "metadata": {}, "outputs": [ From 23eb1a986c5c6fdb956a09704bf079c8f2d0a244 Mon Sep 17 00:00:00 2001 From: George Williams Date: Fri, 13 Sep 2024 13:30:20 -0700 Subject: [PATCH 3/3] removing algos in IGNORE_ALGOS config; rerender markdown --- neurips23/latitude-m4-metal-medium.md | 289 +++++----- neurips23/latitude/analysis.ipynb | 787 ++++++++++++-------------- 2 files changed, 510 insertions(+), 566 deletions(-) diff --git a/neurips23/latitude-m4-metal-medium.md b/neurips23/latitude-m4-metal-medium.md index 864cf3cf..dde7af52 100644 --- a/neurips23/latitude-m4-metal-medium.md +++ b/neurips23/latitude-m4-metal-medium.md @@ -1,190 +1,179 @@ # Eval On AMD 3GHz/16-Core + 125GB RAM + NVMe SSD (Bare Metal) - +
- - - + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - - - - - - - - - - - - + + + + + + + + +
filtersparseoodfiltersparseood
rankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqps
1
zilliz
 213.3K1
zilliz
 34.8K1
scann
 107.4K1
zilliz
 213.3K1
zilliz
 34.8K1
scann
 107.4K
2
pinecone
 146.7K2
pyanns
 26.9K2
pinecone-ood
 76.9K2
pinecone
 146.7K2
pyanns
 26.9K2
pinecone-ood
 76.9K
3
puck
 62.3K3
pinecone_smips
 12.0K3
zilliz
 73.5K3
puck
 62.3K3
pinecone_smips
 12.0K3
zilliz
 73.5K
4
parlayivf
 55.0K4
shnsw
 8.2K4
pyanns
 55.5K4
parlayivf
 55.0K4
shnsw
 8.2K4
pyanns
 55.5K
5
wm_filter
 20.9K5
nle
 2.9K5
sustech-ood
 28.5K5
wm_filter
 20.9K5
nle
 2.9K5
sustech-ood
 28.5K
6
pyanns
 9.0K6
cufe
 0.1K6
mysteryann-dif
 27.9K6
pyanns
 9.0K6
cufe
 0.1K6
mysteryann-dif
 27.9K
7
faissplus
 8.5K7
linscan
 0.1K7
mysteryann
 26.6K7
faissplus
 8.5K7
linscan
 0.1K7
mysteryann
 26.6K
8
faiss
 7.3K
spmat
8
vamana
 20.0K8
faiss
 7.3K
spmat
8
vamana
 20.0K
9
cufe
 6.3K
sustech-whu
9
puck
 19.0K9
cufe
 6.3K
sustech-whu
9
puck
 19.0K
dhq
10
ngt
 11.9K
dhq
10
ngt
 11.9K
fdufilterdiskann
11
epsearch
 7.7K
fdufilterdiskann
11
epsearch
 7.7K
hwtl_sdu_anns_filter
12
diskann
 6.3K
hwtl_sdu_anns_filter
12
diskann
 6.3K
13
cufe
 5.4K13
cufe
 5.4K
hanns
puck-fizz
puck-fizz
diff --git a/neurips23/latitude/analysis.ipynb b/neurips23/latitude/analysis.ipynb index 26ce6266..ac38aa31 100644 --- a/neurips23/latitude/analysis.ipynb +++ b/neurips23/latitude/analysis.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 15, + "execution_count": 31, "id": "21ac7846-3ebf-4f2a-b545-be471d0ce5a8", "metadata": {}, "outputs": [], @@ -27,7 +27,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 32, "id": "936c9c86-f611-4b3f-ae6b-d9d02347a4e2", "metadata": {}, "outputs": [], @@ -46,7 +46,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 33, "id": "071746d6-1cdc-404e-beef-d643596218fd", "metadata": {}, "outputs": [], @@ -58,7 +58,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 34, "id": "7193be1a-a6de-43df-9f60-9243f5ecf473", "metadata": {}, "outputs": [], @@ -70,7 +70,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 35, "id": "5c83a9db-fc3f-4813-91c8-a7a1275b086d", "metadata": {}, "outputs": [], @@ -82,7 +82,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 37, "id": "303b19d1-d0a6-4f0c-9dd0-08879df9cdd6", "metadata": {}, "outputs": [], @@ -116,6 +116,9 @@ " # retrieve all participating track algorithm names via track algo subdirectory \n", " track_dir = os.path.join( COMPETITION_DIR, \"%s/*/Dockerfile\" % track )\n", " algos_participating = [os.path.basename(os.path.dirname(p)) for p in glob.glob( track_dir ) ]\n", + "\n", + " # remove any temp ignores\n", + " algos_participating = list( set(algos_participating ) - set( IGNORE_ALGOS ) ) \n", " \n", " # compute difference of algo lists of track subdirs and algos in results - these did not qualify\n", " algos_did_not_qualify = list( set(algos_participating) - set(list(ranking_df[\"algorithm\"])) )\n", @@ -141,7 +144,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 38, "id": "51cf0a75-1acf-44de-b7f1-2cf030cadd94", "metadata": {}, "outputs": [ @@ -428,21 +431,11 @@ " \n", " \n", " 13\n", - " hanns\n", - " NaN\n", - " NaN\n", - " error\n", - " 14\n", - " ood\n", - " text2image-10M\n", - " \n", - " \n", - " 14\n", " puck-fizz\n", " NaN\n", " NaN\n", " error\n", - " 15\n", + " 14\n", " ood\n", " text2image-10M\n", " \n", @@ -567,8 +560,7 @@ "10 epsearch 7.66 0.92 ok 11 ood \n", "11 diskann 6.30 0.91 ok 12 ood \n", "12 cufe 5.40 0.91 ok 13 ood \n", - "13 hanns NaN NaN error 14 ood \n", - "14 puck-fizz NaN NaN error 15 ood \n", + "13 puck-fizz NaN NaN error 14 ood \n", "0 zilliz 34.76 0.93 ok 1 sparse \n", "1 pyanns 26.86 0.92 ok 2 sparse \n", "2 pinecone_smips 12.00 0.90 ok 3 sparse \n", @@ -606,7 +598,6 @@ "11 text2image-10M \n", "12 text2image-10M \n", "13 text2image-10M \n", - "14 text2image-10M \n", "0 sparse-full \n", "1 sparse-full \n", "2 sparse-full \n", @@ -618,7 +609,7 @@ "8 sparse-full " ] }, - "execution_count": 21, + "execution_count": 38, "metadata": {}, "output_type": "execute_result" } @@ -631,7 +622,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 39, "id": "fbc47e1b-a441-413a-8166-bb834a386cc3", "metadata": {}, "outputs": [ @@ -888,21 +879,11 @@ " \n", " \n", " 13\n", - " hanns\n", - " NaN\n", - " NaN\n", - " error\n", - " 14\n", - " ood\n", - " text2image-10M\n", - " \n", - " \n", - " 14\n", " puck-fizz\n", " NaN\n", " NaN\n", " error\n", - " 15\n", + " 14\n", " ood\n", " text2image-10M\n", " \n", @@ -1054,8 +1035,7 @@ "10 epsearch 7.66 0.92 ok 11 ood \n", "11 diskann 6.30 0.91 ok 12 ood \n", "12 cufe 5.40 0.91 ok 13 ood \n", - "13 hanns NaN NaN error 14 ood \n", - "14 puck-fizz NaN NaN error 15 ood \n", + "13 puck-fizz NaN NaN error 14 ood \n", "0 zilliz 213.29 0.93 ok 1 filter \n", "1 pinecone 146.72 0.92 ok 2 filter \n", "2 puck 62.26 0.94 ok 3 filter \n", @@ -1093,7 +1073,6 @@ "11 text2image-10M \n", "12 text2image-10M \n", "13 text2image-10M \n", - "14 text2image-10M \n", "0 yfcc-10M \n", "1 yfcc-10M \n", "2 yfcc-10M \n", @@ -1108,7 +1087,7 @@ "11 yfcc-10M " ] }, - "execution_count": 22, + "execution_count": 39, "metadata": {}, "output_type": "execute_result" } @@ -1122,7 +1101,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 40, "id": "c90fba89-728a-4f54-bb80-95834bd33005", "metadata": {}, "outputs": [], @@ -1138,7 +1117,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 41, "id": "f4e617b4-112b-45e1-bc3b-2692d980d19f", "metadata": {}, "outputs": [], @@ -1153,7 +1132,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 42, "id": "8d2653b4-f070-4761-8b5e-9a9fa03f19cc", "metadata": {}, "outputs": [], @@ -1166,7 +1145,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 43, "id": "d0accb53-c34b-418e-85a4-253c46cdb90b", "metadata": {}, "outputs": [], @@ -1189,7 +1168,7 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 44, "id": "5cee1395-0e41-4ba4-b586-f9c4b56a9024", "metadata": {}, "outputs": [ @@ -1198,25 +1177,25 @@ "text/html": [ "\n", - "\n", + "
\n", " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", " \n", @@ -1233,193 +1212,181 @@ " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", "
trackfiltersparseoodfiltersparseood
 rankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqps
rank
11zilliz213.2850591zilliz34.7599761scann107.413009
22pinecone146.7212422pyanns26.8585192pinecone-ood76.865423
33puck62.2576953pinecone_smips12.0031823zilliz73.509610
44parlayivf55.0331984shnsw8.2472774pyanns55.504352
55wm_filter20.8748575nle2.9456265sustech-ood28.458263
66pyanns8.9880226cufe0.0849766mysteryann-dif27.946097
77faissplus8.4932647linscan0.0607567mysteryann26.560867
88faiss7.327855spmatnan8vamana19.965517
99cufe6.318462sustech-whunan9puck18.960224
10dhqnannannan10ngt11.921399
11fdufilterdiskannnannannan11epsearch7.663219
12hwtl_sdu_anns_filternannannan12diskann6.295839
13nannannannan13cufe5.404492
14nannannannanhannsnan
15nannannannanpuck-fizznan11zilliz213.2850591zilliz34.7599761scann107.413009
22pinecone146.7212422pyanns26.8585192pinecone-ood76.865423
33puck62.2576953pinecone_smips12.0031823zilliz73.509610
44parlayivf55.0331984shnsw8.2472774pyanns55.504352
55wm_filter20.8748575nle2.9456265sustech-ood28.458263
66pyanns8.9880226cufe0.0849766mysteryann-dif27.946097
77faissplus8.4932647linscan0.0607567mysteryann26.560867
88faiss7.327855spmatnan8vamana19.965517
99cufe6.318462sustech-whunan9puck18.960224
10dhqnannannan10ngt11.921399
11fdufilterdiskannnannannan11epsearch7.663219
12hwtl_sdu_anns_filternannannan12diskann6.295839
13nannannannan13cufe5.404492
14nannannannanpuck-fizznan
\n" ], "text/plain": [ - "" + "" ] }, - "execution_count": 27, + "execution_count": 44, "metadata": {}, "output_type": "execute_result" } @@ -1433,7 +1400,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 45, "id": "884f5113-01a9-42a7-9262-5cea8e9cd751", "metadata": {}, "outputs": [ @@ -1446,197 +1413,185 @@ "WARNING: Using error file path for filter/dhq\n", "WARNING: Using error file path for filter/fdufilterdiskann\n", "WARNING: Using error file path for filter/hwtl_sdu_anns_filter\n", - "WARNING: Ignoring hanns\n", "WARNING: Using error file path for ood/puck-fizz\n" ] }, { "data": { "text/html": [ - "\n", + "
\n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", "
filtersparseoodfiltersparseood
rankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqps
1
zilliz
 213.3K1
zilliz
 34.8K1
scann
 107.4K
2
pinecone
 146.7K2
pyanns
 26.9K2
pinecone-ood
 76.9K
3
puck
 62.3K3
pinecone_smips
 12.0K3
zilliz
 73.5K
4
parlayivf
 55.0K4
shnsw
 8.2K4
pyanns
 55.5K
5
wm_filter
 20.9K5
nle
 2.9K5
sustech-ood
 28.5K
6
pyanns
 9.0K6
cufe
 0.1K6
mysteryann-dif
 27.9K
7
faissplus
 8.5K7
linscan
 0.1K7
mysteryann
 26.6K
8
faiss
 7.3K
spmat
8
vamana
 20.0K
9
cufe
 6.3K
sustech-whu
9
puck
 19.0K
dhq
10
ngt
 11.9K
fdufilterdiskann
11
epsearch
 7.7K
hwtl_sdu_anns_filter
12
diskann
 6.3K
13
cufe
 5.4K
hanns
puck-fizz
1
zilliz
 213.3K1
zilliz
 34.8K1
scann
 107.4K
2
pinecone
 146.7K2
pyanns
 26.9K2
pinecone-ood
 76.9K
3
puck
 62.3K3
pinecone_smips
 12.0K3
zilliz
 73.5K
4
parlayivf
 55.0K4
shnsw
 8.2K4
pyanns
 55.5K
5
wm_filter
 20.9K5
nle
 2.9K5
sustech-ood
 28.5K
6
pyanns
 9.0K6
cufe
 0.1K6
mysteryann-dif
 27.9K
7
faissplus
 8.5K7
linscan
 0.1K7
mysteryann
 26.6K
8
faiss
 7.3K
spmat
8
vamana
 20.0K
9
cufe
 6.3K
sustech-whu
9
puck
 19.0K
dhq
10
ngt
 11.9K
fdufilterdiskann
11
epsearch
 7.7K
hwtl_sdu_anns_filter
12
diskann
 6.3K
13
cufe
 5.4K
puck-fizz
\n" @@ -1645,7 +1600,7 @@ "" ] }, - "execution_count": 28, + "execution_count": 45, "metadata": {}, "output_type": "execute_result" } @@ -1681,7 +1636,7 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 46, "id": "25fc371b-1a14-4fca-97de-acd79070c0ad", "metadata": {}, "outputs": [