Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Bump Faiss commit from 33c0ba5 to 4eecd91 #2194

Merged
merged 2 commits into from
Oct 11, 2024
Merged

Bump Faiss commit from 33c0ba5 to 4eecd91 #2194

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
25ff604
Bump Faiss commit from 33c0ba5 to 4eecd91
naveentatikonda Oct 7, 2024
93a6c28
Update Faiss patches after commit bump
naveentatikonda Oct 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion jni/external/faiss
Open in desktop
Submodule faiss updated 186 files
46 changes: 22 additions & 24 deletions jni/patches/faiss/0001-Custom-patch-to-support-multi-vector.patch
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
From 9e5affabe2caacf38f5585a0b906620dd35deef5 Mon Sep 17 00:00:00 2001
From e775a8e65da96232822d5aed77f538592fccffda Mon Sep 17 00:00:00 2001
From: Heemin Kim <[email protected]>
Date: Tue, 30 Jan 2024 14:43:56 -0800
Subject: [PATCH] Add IDGrouper for HNSW

Signed-off-by: Heemin Kim <[email protected]>
---
faiss/CMakeLists.txt | 3 +
faiss/Index.h | 8 +-
faiss/Index.h | 6 +-
faiss/IndexHNSW.cpp | 13 +-
faiss/IndexIDMap.cpp | 29 +++++
faiss/IndexIDMap.h | 22 ++++
Expand All @@ -18,34 +18,34 @@ Signed-off-by: Heemin Kim <[email protected]>
tests/CMakeLists.txt | 2 +
tests/test_group_heap.cpp | 98 +++++++++++++++
tests/test_id_grouper.cpp | 241 +++++++++++++++++++++++++++++++++++++
13 files changed, 890 insertions(+), 5 deletions(-)
13 files changed, 889 insertions(+), 4 deletions(-)
create mode 100644 faiss/impl/IDGrouper.cpp
create mode 100644 faiss/impl/IDGrouper.h
create mode 100644 faiss/utils/GroupHeap.h
create mode 100644 tests/test_group_heap.cpp
create mode 100644 tests/test_id_grouper.cpp

diff --git a/faiss/CMakeLists.txt b/faiss/CMakeLists.txt
index 1b0860f3..f3d72df3 100644
index 2871d974..d0bcec6a 100644
--- a/faiss/CMakeLists.txt
+++ b/faiss/CMakeLists.txt
@@ -54,6 +54,7 @@ set(FAISS_SRC
@@ -55,6 +55,7 @@ set(FAISS_SRC
impl/AuxIndexStructures.cpp
impl/CodePacker.cpp
impl/IDSelector.cpp
+ impl/IDGrouper.cpp
impl/FaissException.cpp
impl/HNSW.cpp
impl/NSG.cpp
@@ -149,6 +150,7 @@ set(FAISS_HEADERS
@@ -151,6 +152,7 @@ set(FAISS_HEADERS
impl/AuxIndexStructures.h
impl/CodePacker.h
impl/IDSelector.h
+ impl/IDGrouper.h
impl/DistanceComputer.h
impl/FaissAssert.h
impl/FaissException.h
@@ -184,6 +186,7 @@ set(FAISS_HEADERS
@@ -186,6 +188,7 @@ set(FAISS_HEADERS
invlists/InvertedListsIOHook.h
utils/AlignedTable.h
utils/bf16.h
Expand All @@ -54,23 +54,21 @@ index 1b0860f3..f3d72df3 100644
utils/WorkerThread.h
utils/distances.h
diff --git a/faiss/Index.h b/faiss/Index.h
index 3d1bdb99..a8622858 100644
index f57140ec..8f511e5d 100644
--- a/faiss/Index.h
+++ b/faiss/Index.h
@@ -38,9 +38,10 @@

@@ -51,8 +51,9 @@
namespace faiss {

-/// Forward declarations see impl/AuxIndexStructures.h, impl/IDSelector.h and
-/// impl/DistanceComputer.h
+/// Forward declarations see impl/AuxIndexStructures.h, impl/IDSelector.h
/// Forward declarations see impl/AuxIndexStructures.h, impl/IDSelector.h
-/// and impl/DistanceComputer.h
+/// ,impl/IDGrouper.h and impl/DistanceComputer.h
struct IDSelector;
+struct IDGrouper;
struct RangeSearchResult;
struct DistanceComputer;

@@ -52,6 +53,9 @@ struct DistanceComputer;
@@ -64,6 +65,9 @@ struct DistanceComputer;
struct SearchParameters {
/// if non-null, only these IDs will be considered during search.
IDSelector* sel = nullptr;
Expand All @@ -81,10 +79,10 @@ index 3d1bdb99..a8622858 100644
virtual ~SearchParameters() {}
};
diff --git a/faiss/IndexHNSW.cpp b/faiss/IndexHNSW.cpp
index 8e5c654f..d473b6ad 100644
index 6a1186ca..9c8a8255 100644
--- a/faiss/IndexHNSW.cpp
+++ b/faiss/IndexHNSW.cpp
@@ -320,10 +320,17 @@ void IndexHNSW::search(
@@ -301,10 +301,17 @@ void IndexHNSW::search(
const SearchParameters* params_in) const {
FAISS_THROW_IF_NOT(k > 0);

Expand Down Expand Up @@ -198,10 +196,10 @@ index 2d164123..a68887bd 100644
+
} // namespace faiss
diff --git a/faiss/impl/HNSW.cpp b/faiss/impl/HNSW.cpp
index 3ba5f72f..c574ce39 100644
index c3693fd9..7ae28062 100644
--- a/faiss/impl/HNSW.cpp
+++ b/faiss/impl/HNSW.cpp
@@ -831,6 +831,12 @@ int extract_k_from_ResultHandler(ResultHandler<C>& res) {
@@ -906,6 +906,12 @@ int extract_k_from_ResultHandler(ResultHandler<C>& res) {
if (auto hres = dynamic_cast<RH::SingleResultHandler*>(&res)) {
return hres->k;
}
Expand Down Expand Up @@ -329,19 +327,19 @@ index 00000000..d56113d9
+
+} // namespace faiss
diff --git a/faiss/impl/ResultHandler.h b/faiss/impl/ResultHandler.h
index 713fe8e4..d307fd70 100644
index 3116eb24..126ed015 100644
--- a/faiss/impl/ResultHandler.h
+++ b/faiss/impl/ResultHandler.h
@@ -13,6 +13,8 @@

@@ -14,6 +14,8 @@
#include <faiss/impl/AuxIndexStructures.h>
#include <faiss/impl/FaissException.h>
#include <faiss/impl/IDSelector.h>
+#include <faiss/impl/IDGrouper.h>
+#include <faiss/utils/GroupHeap.h>
#include <faiss/utils/Heap.h>
#include <faiss/utils/partitioning.h>
#include <iostream>
@@ -267,6 +269,193 @@ struct HeapBlockResultHandler : BlockResultHandler<C> {

@@ -286,6 +288,193 @@ struct HeapBlockResultHandler : BlockResultHandler<C, use_sel> {
}
};

Expand Down Expand Up @@ -725,7 +723,7 @@ index 00000000..3b7078da
+} // namespace faiss
\ No newline at end of file
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 3980d7dd..c888a5a6 100644
index c41edf0c..87ab2020 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -27,6 +27,8 @@ set(FAISS_TEST_SRC
Expand Down
22 changes: 11 additions & 11 deletions jni/patches/faiss/0002-Enable-precomp-table-to-be-shared-ivfpq.patch
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
From a33e6ef35385009f24200586294f96235cb95d61 Mon Sep 17 00:00:00 2001
From 9b33874562c9e62abf4a863657c54f0d349b0f67 Mon Sep 17 00:00:00 2001
From: John Mazanec <[email protected]>
Date: Wed, 21 Feb 2024 15:34:15 -0800
Subject: [PATCH] Enable precomp table to be shared ivfpq
Expand All @@ -22,7 +22,7 @@ Signed-off-by: John Mazanec <[email protected]>
create mode 100644 tests/test_ivfpq_share_table.cpp

diff --git a/faiss/IndexIVFPQ.cpp b/faiss/IndexIVFPQ.cpp
index 0b7f4d05..07bc7e83 100644
index 100f499c..09508890 100644
--- a/faiss/IndexIVFPQ.cpp
+++ b/faiss/IndexIVFPQ.cpp
@@ -59,6 +59,29 @@ IndexIVFPQ::IndexIVFPQ(
Expand Down Expand Up @@ -55,7 +55,7 @@ index 0b7f4d05..07bc7e83 100644
}

/****************************************************************
@@ -466,11 +489,23 @@ void IndexIVFPQ::precompute_table() {
@@ -464,11 +487,23 @@ void IndexIVFPQ::precompute_table() {
use_precomputed_table,
quantizer,
pq,
Expand All @@ -80,7 +80,7 @@ index 0b7f4d05..07bc7e83 100644
namespace {

#define TIC t0 = get_cycles()
@@ -650,7 +685,7 @@ struct QueryTables {
@@ -648,7 +683,7 @@ struct QueryTables {

fvec_madd(
pq.M * pq.ksub,
Expand All @@ -89,7 +89,7 @@ index 0b7f4d05..07bc7e83 100644
-2.0,
sim_table_2,
sim_table);
@@ -679,7 +714,7 @@ struct QueryTables {
@@ -677,7 +712,7 @@ struct QueryTables {
k >>= cpq.nbits;

// get corresponding table
Expand All @@ -98,7 +98,7 @@ index 0b7f4d05..07bc7e83 100644
(ki * pq.M + cm * Mf) * pq.ksub;

if (polysemous_ht == 0) {
@@ -709,7 +744,7 @@ struct QueryTables {
@@ -707,7 +742,7 @@ struct QueryTables {
dis0 = coarse_dis;

const float* s =
Expand All @@ -107,7 +107,7 @@ index 0b7f4d05..07bc7e83 100644
for (int m = 0; m < pq.M; m++) {
sim_table_ptrs[m] = s;
s += pq.ksub;
@@ -729,7 +764,7 @@ struct QueryTables {
@@ -727,7 +762,7 @@ struct QueryTables {
int ki = k & ((uint64_t(1) << cpq.nbits) - 1);
k >>= cpq.nbits;

Expand All @@ -116,7 +116,7 @@ index 0b7f4d05..07bc7e83 100644
(ki * pq.M + cm * Mf) * pq.ksub;

for (int m = m0; m < m0 + Mf; m++) {
@@ -1346,6 +1381,8 @@ IndexIVFPQ::IndexIVFPQ() {
@@ -1344,6 +1379,8 @@ IndexIVFPQ::IndexIVFPQ() {
do_polysemous_training = false;
polysemous_ht = 0;
polysemous_training = nullptr;
Expand Down Expand Up @@ -302,13 +302,13 @@ index 00dd2f11..91f35a6e 100644
/// same as the regular IVFPQ encoder. The codes are not reorganized by
/// blocks a that point
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index c888a5a6..83ecedfd 100644
index 87ab2020..a859516c 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -37,6 +37,7 @@ set(FAISS_TEST_SRC
test_disable_pq_sdc_tables.cpp
@@ -38,6 +38,7 @@ set(FAISS_TEST_SRC
test_common_ivf_empty_index.cpp
test_callback.cpp
test_utils.cpp
+ test_ivfpq_share_table.cpp
)

Expand Down
Loading