-
Notifications
You must be signed in to change notification settings - Fork 654
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #523 from nmslib/develop
Release v0.8.0
- Loading branch information
Showing
23 changed files
with
1,190 additions
and
90 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -10,3 +10,4 @@ var/ | |
| .vscode/ | ||
| .vs/ | ||
| **.DS_Store | ||
| *.pyc | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,66 @@ | ||
| #include "../../hnswlib/hnswlib.h" | ||
|
|
||
| typedef unsigned int docidtype; | ||
| typedef float dist_t; | ||
|
|
||
| int main() { | ||
| int dim = 16; // Dimension of the elements | ||
| int max_elements = 10000; // Maximum number of elements, should be known beforehand | ||
| int M = 16; // Tightly connected with internal dimensionality of the data | ||
| // strongly affects the memory consumption | ||
| int ef_construction = 200; // Controls index search speed/build speed tradeoff | ||
| int min_num_candidates = 100; // Minimum number of candidates to search in the epsilon region | ||
| // this parameter is similar to ef | ||
|
|
||
| int num_queries = 5; | ||
| float epsilon2 = 2.0; // Squared distance to query | ||
|
|
||
| // Initing index | ||
| hnswlib::L2Space space(dim); | ||
| hnswlib::HierarchicalNSW<dist_t>* alg_hnsw = new hnswlib::HierarchicalNSW<dist_t>(&space, max_elements, M, ef_construction); | ||
|
|
||
| // Generate random data | ||
| std::mt19937 rng; | ||
| rng.seed(47); | ||
| std::uniform_real_distribution<> distrib_real; | ||
|
|
||
| size_t data_point_size = space.get_data_size(); | ||
| char* data = new char[data_point_size * max_elements]; | ||
| for (int i = 0; i < max_elements; i++) { | ||
| char* point_data = data + i * data_point_size; | ||
| for (int j = 0; j < dim; j++) { | ||
| char* vec_data = point_data + j * sizeof(float); | ||
| float value = distrib_real(rng); | ||
| *(float*)vec_data = value; | ||
| } | ||
| } | ||
|
|
||
| // Add data to index | ||
| for (int i = 0; i < max_elements; i++) { | ||
| hnswlib::labeltype label = i; | ||
| char* point_data = data + i * data_point_size; | ||
| alg_hnsw->addPoint(point_data, label); | ||
| } | ||
|
|
||
| // Query random vectors | ||
| for (int i = 0; i < num_queries; i++) { | ||
| char* query_data = new char[data_point_size]; | ||
| for (int j = 0; j < dim; j++) { | ||
| size_t offset = j * sizeof(float); | ||
| char* vec_data = query_data + offset; | ||
| float value = distrib_real(rng); | ||
| *(float*)vec_data = value; | ||
| } | ||
| std::cout << "Query #" << i << "\n"; | ||
| hnswlib::EpsilonSearchStopCondition<dist_t> stop_condition(epsilon2, min_num_candidates, max_elements); | ||
| std::vector<std::pair<float, hnswlib::labeltype>> result = | ||
| alg_hnsw->searchStopConditionClosest(query_data, stop_condition); | ||
| size_t num_vectors = result.size(); | ||
| std::cout << "Found " << num_vectors << " vectors\n"; | ||
| delete[] query_data; | ||
| } | ||
|
|
||
| delete[] data; | ||
| delete alg_hnsw; | ||
| return 0; | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,83 @@ | ||
| #include "../../hnswlib/hnswlib.h" | ||
|
|
||
| typedef unsigned int docidtype; | ||
| typedef float dist_t; | ||
|
|
||
| int main() { | ||
| int dim = 16; // Dimension of the elements | ||
| int max_elements = 10000; // Maximum number of elements, should be known beforehand | ||
| int M = 16; // Tightly connected with internal dimensionality of the data | ||
| // strongly affects the memory consumption | ||
| int ef_construction = 200; // Controls index search speed/build speed tradeoff | ||
|
|
||
| int num_queries = 5; | ||
| int num_docs = 5; // Number of documents to search | ||
| int ef_collection = 6; // Number of candidate documents during the search | ||
| // Controlls the recall: higher ef leads to better accuracy, but slower search | ||
| docidtype min_doc_id = 0; | ||
| docidtype max_doc_id = 9; | ||
|
|
||
| // Initing index | ||
| hnswlib::MultiVectorL2Space<docidtype> space(dim); | ||
| hnswlib::HierarchicalNSW<dist_t>* alg_hnsw = new hnswlib::HierarchicalNSW<dist_t>(&space, max_elements, M, ef_construction); | ||
|
|
||
| // Generate random data | ||
| std::mt19937 rng; | ||
| rng.seed(47); | ||
| std::uniform_real_distribution<> distrib_real; | ||
| std::uniform_int_distribution<docidtype> distrib_docid(min_doc_id, max_doc_id); | ||
|
|
||
| size_t data_point_size = space.get_data_size(); | ||
| char* data = new char[data_point_size * max_elements]; | ||
| for (int i = 0; i < max_elements; i++) { | ||
| // set vector value | ||
| char* point_data = data + i * data_point_size; | ||
| for (int j = 0; j < dim; j++) { | ||
| char* vec_data = point_data + j * sizeof(float); | ||
| float value = distrib_real(rng); | ||
| *(float*)vec_data = value; | ||
| } | ||
| // set document id | ||
| docidtype doc_id = distrib_docid(rng); | ||
| space.set_doc_id(point_data, doc_id); | ||
| } | ||
|
|
||
| // Add data to index | ||
| std::unordered_map<hnswlib::labeltype, docidtype> label_docid_lookup; | ||
| for (int i = 0; i < max_elements; i++) { | ||
| hnswlib::labeltype label = i; | ||
| char* point_data = data + i * data_point_size; | ||
| alg_hnsw->addPoint(point_data, label); | ||
| label_docid_lookup[label] = space.get_doc_id(point_data); | ||
| } | ||
|
|
||
| // Query random vectors | ||
| size_t query_size = dim * sizeof(float); | ||
| for (int i = 0; i < num_queries; i++) { | ||
| char* query_data = new char[query_size]; | ||
| for (int j = 0; j < dim; j++) { | ||
| size_t offset = j * sizeof(float); | ||
| char* vec_data = query_data + offset; | ||
| float value = distrib_real(rng); | ||
| *(float*)vec_data = value; | ||
| } | ||
| std::cout << "Query #" << i << "\n"; | ||
| hnswlib::MultiVectorSearchStopCondition<docidtype, dist_t> stop_condition(space, num_docs, ef_collection); | ||
| std::vector<std::pair<float, hnswlib::labeltype>> result = | ||
| alg_hnsw->searchStopConditionClosest(query_data, stop_condition); | ||
| size_t num_vectors = result.size(); | ||
|
|
||
| std::unordered_map<docidtype, size_t> doc_counter; | ||
| for (auto pair: result) { | ||
| hnswlib::labeltype label = pair.second; | ||
| docidtype doc_id = label_docid_lookup[label]; | ||
| doc_counter[doc_id] += 1; | ||
| } | ||
| std::cout << "Found " << doc_counter.size() << " documents, " << num_vectors << " vectors\n"; | ||
| delete[] query_data; | ||
| } | ||
|
|
||
| delete[] data; | ||
| delete alg_hnsw; | ||
| return 0; | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.