Skip to content

Commit

Permalink
feat: Added gte-large, nomic-text 1.5, cleanup
Browse files Browse the repository at this point in the history
  • Loading branch information
@Anush008
Anush008 committed Feb 21, 2024
1 parent 38c4eb1 commit f2f09fb
Show file tree
Hide file tree
Showing 6 changed files with 87 additions and 264 deletions.
94 changes: 59 additions & 35 deletions docs/examples/Supported_Models.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
},
{
"cell_type": "code",
"execution_count": 2,
"execution_count": 6,
"metadata": {},
"outputs": [
{
Expand Down Expand Up @@ -110,46 +110,62 @@
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2</td>\n",
" <td>384</td>\n",
" <td>Sentence Transformer model, paraphrase-multilingual-MiniLM-L12-v2</td>\n",
" <td>0.46</td>\n",
" <td>{'hf': 'qdrant/paraphrase-multilingual-MiniLM-L12-v2-onnx-Q'}</td>\n",
" <td>nomic-ai/nomic-embed-text-v1</td>\n",
" <td>768</td>\n",
" <td>8192 context length english model</td>\n",
" <td>0.54</td>\n",
" <td>{'hf': 'nomic-ai/nomic-embed-text-v1'}</td>\n",
" </tr>\n",
" <tr>\n",
" <th>9</th>\n",
" <td>nomic-ai/nomic-embed-text-v1</td>\n",
" <td>nomic-ai/nomic-embed-text-v1.5</td>\n",
" <td>768</td>\n",
" <td>8192 context length english model</td>\n",
" <td>0.54</td>\n",
" <td>{'hf': 'nomic-ai/nomic-embed-text-v1'}</td>\n",
" <td>{'hf': 'nomic-ai/nomic-embed-text-v1.5'}</td>\n",
" </tr>\n",
" <tr>\n",
" <th>10</th>\n",
" <td>thenlper/gte-large</td>\n",
" <td>1024</td>\n",
" <td>Large general text embeddings model</td>\n",
" <td>1.34</td>\n",
" <td>{'hf': 'qdrant/gte-large-onnx'}</td>\n",
" </tr>\n",
" <tr>\n",
" <th>11</th>\n",
" <td>intfloat/multilingual-e5-large</td>\n",
" <td>1024</td>\n",
" <td>Multilingual model, e5-large. Recommend using this model for non-English languages</td>\n",
" <td>2.24</td>\n",
" <td>{'url': 'https://storage.googleapis.com/qdrant-fastembed/fast-multilingual-e5-large.tar.gz', 'hf': 'qdrant/multilingual-e5-large-onnx'}</td>\n",
" </tr>\n",
" <tr>\n",
" <th>11</th>\n",
" <th>12</th>\n",
" <td>sentence-transformers/paraphrase-multilingual-mpnet-base-v2</td>\n",
" <td>768</td>\n",
" <td>Sentence-transformers model for tasks like clustering or semantic search</td>\n",
" <td>1.11</td>\n",
" <td>{'hf': 'xenova/paraphrase-multilingual-mpnet-base-v2'}</td>\n",
" </tr>\n",
" <tr>\n",
" <th>12</th>\n",
" <th>13</th>\n",
" <td>sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2</td>\n",
" <td>384</td>\n",
" <td>Sentence Transformer model, paraphrase-multilingual-MiniLM-L12-v2</td>\n",
" <td>0.46</td>\n",
" <td>{'hf': 'qdrant/paraphrase-multilingual-MiniLM-L12-v2-onnx-Q'}</td>\n",
" </tr>\n",
" <tr>\n",
" <th>14</th>\n",
" <td>jinaai/jina-embeddings-v2-base-en</td>\n",
" <td>768</td>\n",
" <td>English embedding model supporting 8192 sequence length</td>\n",
" <td>0.55</td>\n",
" <td>{'hf': 'xenova/jina-embeddings-v2-base-en'}</td>\n",
" </tr>\n",
" <tr>\n",
" <th>13</th>\n",
" <th>15</th>\n",
" <td>jinaai/jina-embeddings-v2-small-en</td>\n",
" <td>512</td>\n",
" <td>English embedding model supporting 8192 sequence length</td>\n",
Expand All @@ -170,12 +186,14 @@
"5 BAAI/bge-small-en-v1.5 384 \n",
"6 BAAI/bge-small-zh-v1.5 512 \n",
"7 sentence-transformers/all-MiniLM-L6-v2 384 \n",
"8 sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2 384 \n",
"9 nomic-ai/nomic-embed-text-v1 768 \n",
"10 intfloat/multilingual-e5-large 1024 \n",
"11 sentence-transformers/paraphrase-multilingual-mpnet-base-v2 768 \n",
"12 jinaai/jina-embeddings-v2-base-en 768 \n",
"13 jinaai/jina-embeddings-v2-small-en 512 \n",
"8 nomic-ai/nomic-embed-text-v1 768 \n",
"9 nomic-ai/nomic-embed-text-v1.5 768 \n",
"10 thenlper/gte-large 1024 \n",
"11 intfloat/multilingual-e5-large 1024 \n",
"12 sentence-transformers/paraphrase-multilingual-mpnet-base-v2 768 \n",
"13 sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2 384 \n",
"14 jinaai/jina-embeddings-v2-base-en 768 \n",
"15 jinaai/jina-embeddings-v2-small-en 512 \n",
"\n",
" description \\\n",
"0 Base English model \n",
Expand All @@ -186,12 +204,14 @@
"5 Fast and Default English model \n",
"6 Fast and recommended Chinese model \n",
"7 Sentence Transformer model, MiniLM-L6-v2 \n",
"8 Sentence Transformer model, paraphrase-multilingual-MiniLM-L12-v2 \n",
"8 8192 context length english model \n",
"9 8192 context length english model \n",
"10 Multilingual model, e5-large. Recommend using this model for non-English languages \n",
"11 Sentence-transformers model for tasks like clustering or semantic search \n",
"12 English embedding model supporting 8192 sequence length \n",
"13 English embedding model supporting 8192 sequence length \n",
"10 Large general text embeddings model \n",
"11 Multilingual model, e5-large. Recommend using this model for non-English languages \n",
"12 Sentence-transformers model for tasks like clustering or semantic search \n",
"13 Sentence Transformer model, paraphrase-multilingual-MiniLM-L12-v2 \n",
"14 English embedding model supporting 8192 sequence length \n",
"15 English embedding model supporting 8192 sequence length \n",
"\n",
" size_in_GB \\\n",
"0 0.50 \n",
Expand All @@ -202,12 +222,14 @@
"5 0.13 \n",
"6 0.10 \n",
"7 0.09 \n",
"8 0.46 \n",
"8 0.54 \n",
"9 0.54 \n",
"10 2.24 \n",
"11 1.11 \n",
"12 0.55 \n",
"13 0.13 \n",
"10 1.34 \n",
"11 2.24 \n",
"12 1.11 \n",
"13 0.46 \n",
"14 0.55 \n",
"15 0.13 \n",
"\n",
" sources \n",
"0 {'url': 'https://storage.googleapis.com/qdrant-fastembed/fast-bge-base-en.tar.gz'} \n",
Expand All @@ -218,15 +240,17 @@
"5 {'url': 'https://storage.googleapis.com/qdrant-fastembed/fast-bge-small-en-v1.5.tar.gz', 'hf': 'qdrant/bge-small-en-v1.5-onnx-q'} \n",
"6 {'url': 'https://storage.googleapis.com/qdrant-fastembed/fast-bge-small-zh-v1.5.tar.gz'} \n",
"7 {'url': 'https://storage.googleapis.com/qdrant-fastembed/sentence-transformers-all-MiniLM-L6-v2.tar.gz', 'hf': 'qdrant/all-MiniLM-L6-v2-onnx'} \n",
"8 {'hf': 'qdrant/paraphrase-multilingual-MiniLM-L12-v2-onnx-Q'} \n",
"9 {'hf': 'nomic-ai/nomic-embed-text-v1'} \n",
"10 {'url': 'https://storage.googleapis.com/qdrant-fastembed/fast-multilingual-e5-large.tar.gz', 'hf': 'qdrant/multilingual-e5-large-onnx'} \n",
"11 {'hf': 'xenova/paraphrase-multilingual-mpnet-base-v2'} \n",
"12 {'hf': 'xenova/jina-embeddings-v2-base-en'} \n",
"13 {'hf': 'xenova/jina-embeddings-v2-small-en'} "
"8 {'hf': 'nomic-ai/nomic-embed-text-v1'} \n",
"9 {'hf': 'nomic-ai/nomic-embed-text-v1.5'} \n",
"10 {'hf': 'qdrant/gte-large-onnx'} \n",
"11 {'url': 'https://storage.googleapis.com/qdrant-fastembed/fast-multilingual-e5-large.tar.gz', 'hf': 'qdrant/multilingual-e5-large-onnx'} \n",
"12 {'hf': 'xenova/paraphrase-multilingual-mpnet-base-v2'} \n",
"13 {'hf': 'qdrant/paraphrase-multilingual-MiniLM-L12-v2-onnx-Q'} \n",
"14 {'hf': 'xenova/jina-embeddings-v2-base-en'} \n",
"15 {'hf': 'xenova/jina-embeddings-v2-small-en'} "
]
},
"execution_count": 2,
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
Expand Down Expand Up @@ -256,7 +280,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.4"
"version": "3.10.13"
},
"orig_nbformat": 4
},
Expand Down
143 changes: 0 additions & 143 deletions fastembed/models.json
View file

This file was deleted.

9 changes: 9 additions & 0 deletions fastembed/text/e5_onnx_embedding.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,15 @@
"hf": "xenova/paraphrase-multilingual-mpnet-base-v2",
},
},
{
"model": "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2",
"dim": 384,
"description": "Sentence Transformer model, paraphrase-multilingual-MiniLM-L12-v2",
"size_in_GB": 0.46,
"sources": {
"hf": "qdrant/paraphrase-multilingual-MiniLM-L12-v2-onnx-Q",
},
},
]


Expand Down
23 changes: 16 additions & 7 deletions fastembed/text/onnx_embedding.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,21 +99,30 @@
},
},
{
"model": "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2",
"dim": 384,
"description": "Sentence Transformer model, paraphrase-multilingual-MiniLM-L12-v2",
"size_in_GB": 0.46,
"model": "nomic-ai/nomic-embed-text-v1",
"dim": 768,
"description": "8192 context length english model",
"size_in_GB": 0.54,
"sources": {
"hf": "qdrant/paraphrase-multilingual-MiniLM-L12-v2-onnx-Q",
"hf": "nomic-ai/nomic-embed-text-v1",
},
},
{
"model": "nomic-ai/nomic-embed-text-v1",
"model": "nomic-ai/nomic-embed-text-v1.5",
"dim": 768,
"description": "8192 context length english model",
"size_in_GB": 0.54,
"sources": {
"hf": "nomic-ai/nomic-embed-text-v1",
"hf": "nomic-ai/nomic-embed-text-v1.5",
},
},
{
"model": "thenlper/gte-large",
"dim": 1024,
"description": "Large general text embeddings model",
"size_in_GB": 1.34,
"sources": {
"hf": "qdrant/gte-large-onnx",
},
},
# {
Expand Down
Loading

0 comments on commit f2f09fb

Please sign in to comment.