Make fields searchable in vector DB

[d3buggerdan]

@dosu
When I make this index, the only fields that are searchable are chunk and embedding, but i want to make all of these also searchable but they are only retrievable and filterable right now : "date": ("date", MetadataIndexFieldType.STRING),
"userid": ("userid", MetadataIndexFieldType.STRING),
"contractid": ("contractid", MetadataIndexFieldType.STRING),
"title": ("title", MetadataIndexFieldType.STRING),
"clauseid": ("clauseid", MetadataIndexFieldType.STRING),
"summary": ("summary", MetadataIndexFieldType.STRING),
"keywords": ("keywords", MetadataIndexFieldType.STRING)

metadata_fields = {
"date": ("date", MetadataIndexFieldType.STRING),
"userid": ("userid", MetadataIndexFieldType.STRING),
"contractid": ("contractid", MetadataIndexFieldType.STRING),
"title": ("title", MetadataIndexFieldType.STRING),
"clauseid": ("clauseid", MetadataIndexFieldType.STRING),
"summary": ("summary", MetadataIndexFieldType.STRING),
"keywords": ("keywords", MetadataIndexFieldType.STRING)
}

Define the vector store

vector_store = AzureAISearchVectorStore(
search_or_index_client=index_client,
filterable_metadata_field_keys=metadata_fields,
index_name=index_name,
index_management=IndexManagement.CREATE_IF_NOT_EXISTS,
id_field_key="id",
chunk_field_key="chunk",
embedding_field_key="embedding",
embedding_dimensionality=1536,
metadata_string_field_key="metadata",
doc_id_field_key="doc_id",
language_analyzer="en.lucene",
vector_algorithm_type="exhaustiveKnn",
)

Function to generate a summary for a clause using the LLM

def summarize_clause(llm, clause_text):
prompt = (
"prompt"
)
response = llm.complete(prompt)
summary = response.text.strip()
return summary

Function to generate keywords for a clause using the LLM

def generate_keywords(llm, clause_text):
prompt = (
"prompt"
)
response = llm.complete(prompt)

# Process and clean up the keywords list
keywords = response.text.strip().split(', ')
keywords = [keyword.strip().lower() for keyword in keywords if len(keyword.split()) < 3]  # Remove long phrases
return ', '.join(keywords[:10])  # Ensure the list is exactly 10 keywords, joined into a single string

Create document objects using the clauses from the contract

documents = []
for i, item in enumerate(contract_data["contract"]): # Fixed to use "document"
clause_text = item["clause"]
clause_id = f"clause_{i+1}" # Fixed clause ID to use a proper format

# Generate a summary and keywords for the clause
clause_summary = summarize_clause(llm, clause_text)
clause_keywords = generate_keywords(llm, clause_text)

# Create a document with the summary and keywords included in the metadata
documents.append(
    Document(
        text=clause_text, 
        id_=clause_id, 
        metadata={
            "date": "<REPLACE_WITH_DATE>",  # Replace with actual date
            "userid": "<REPLACE_WITH_USERID>",  # Replace with actual user ID
            "contractid": "<REPLACE_WITH_CONTRACTID>",  # Replace with actual contract ID
            "title": "<REPLACE_WITH_TITLE>",  # Replace with actual title
            "clauseid": clause_id,
            "summary": clause_summary,  # Add the summary to the metadata
            "keywords": clause_keywords  # Add the keywords to the metadata
        }
    )
)

Create the vector store index with the summarized documents

storage_context = StorageContext.from_defaults(vector_store=vector_store)

Settings.llm = llm
Settings.embed_model = embed_model
index = VectorStoreIndex.from_documents(
documents, storage_context=storage_context
)

[dosubot[bot]]

To make the specified fields ("date", "userid", "contractid", "title", "clauseid", "summary", "keywords") searchable in the AzureAISearchVectorStore, you need to ensure that these fields are defined as searchable in the index configuration. Here is how you can modify your setup to include these fields as searchable:

1. Update the AzureAISearchVectorStore Configuration:
   Ensure that the fields are included in the filterable_metadata_field_keys and are also marked as searchable in the index creation process.

2. Modify the _create_index Method:
   Update the _create_index method to include these fields as SearchableField instead of SimpleField.

Here is the updated code:

from azure.search.documents.indexes.models import (
    SearchIndex,
    SimpleField,
    SearchableField,
    SearchField,
    SearchFieldDataType,
    VectorSearch,
    VectorSearchAlgorithmKind,
    VectorSearchAlgorithmMetric,
    HnswAlgorithmConfiguration,
    HnswParameters,
    ExhaustiveKnnAlgorithmConfiguration,
    ExhaustiveKnnParameters,
    VectorSearchProfile,
    SemanticConfiguration,
    SemanticField,
    SemanticPrioritizedFields,
    SemanticSearch,
)

def _create_index(self, index_name: Optional[str]) -> None:
    fields = [
        SimpleField(name=self._field_mapping["id"], type="Edm.String", key=True),
        SearchableField(
            name=self._field_mapping["chunk"],
            type="Edm.String",
            analyzer_name=self._language_analyzer,
        ),
        SearchField(
            name=self._field_mapping["embedding"],
            type=SearchFieldDataType.Collection(SearchFieldDataType.Single),
            searchable=True,
            vector_search_dimensions=self._embedding_dimensionality,
            vector_search_profile_name=self._vector_profile_name,
        ),
        SearchableField(name=self._field_mapping["metadata"], type="Edm.String"),
        SimpleField(
            name=self._field_mapping["doc_id"], type="Edm.String", filterable=True
        ),
    ]

    # Add metadata fields as searchable fields
    metadata_index_fields = [
        SearchableField(name="date", type="Edm.String"),
        SearchableField(name="userid", type="Edm.String"),
        SearchableField(name="contractid", type="Edm.String"),
        SearchableField(name="title", type="Edm.String"),
        SearchableField(name="clauseid", type="Edm.String"),
        SearchableField(name="summary", type="Edm.String"),
        SearchableField(name="keywords", type="Edm.String"),
    ]
    fields.extend(metadata_index_fields)

    vector_search = VectorSearch(
        algorithms=[
            HnswAlgorithmConfiguration(
                name="myHnsw",
                kind=VectorSearchAlgorithmKind.HNSW,
                parameters=HnswParameters(
                    m=4,
                    ef_construction=400,
                    ef_search=500,
                    metric=VectorSearchAlgorithmMetric.COSINE,
                ),
            ),
            ExhaustiveKnnAlgorithmConfiguration(
                name="myExhaustiveKnn",
                kind=VectorSearchAlgorithmKind.EXHAUSTIVE_KNN,
                parameters=ExhaustiveKnnParameters(
                    metric=VectorSearchAlgorithmMetric.COSINE,
                ),
            ),
        ],
        profiles=[
            VectorSearchProfile(
                name="myHnswProfile",
                algorithm_configuration_name="myHnsw",
            ),
            VectorSearchProfile(
                name="myExhaustiveKnnProfile",
                algorithm_configuration_name="myExhaustiveKnn",
            ),
        ],
    )

    semantic_config = SemanticConfiguration(
        name="mySemanticConfig",
        prioritized_fields=SemanticPrioritizedFields(
            content_fields=[SemanticField(field_name=self._field_mapping["chunk"])],
        ),
    )
    semantic_search = SemanticSearch(configurations=[semantic_config])

    index = SearchIndex(
        name=index_name,
        fields=fields,
        vector_search=vector_search,
        semantic_search=semantic_search,
    )

    self._index_client.create_index(index)

This modification ensures that the fields "date", "userid", "contractid", "title", "clauseid", "summary", and "keywords" are searchable in the vector database ^[1][2].

To continue talking to Dosu, mention @dosu.

[dosubot[bot]]

To fix the code and ensure that all specified fields are searchable in the Azure Search index, you need to make sure that the SearchableField and SearchField classes are used correctly. Here is the corrected code:

from azure.search.documents.indexes.models import (
    SimpleField, SearchableField, SearchIndex, VectorSearch, HnswAlgorithmConfiguration,
    VectorSearchAlgorithmKind, VectorSearchAlgorithmMetric, VectorSearchProfile, SemanticConfiguration,
    SemanticPrioritizedFields, SemanticField, SemanticSearch, SearchFieldDataType, SearchField
)

def create_search_index(index_name, index_client):
    try:
        fields = [
            SimpleField(name="id", type="Edm.String", key=True),
            SearchableField(name="chunk", type="Edm.String", analyzer_name="en.lucene"),
            SearchField(
                name="embedding",
                type=SearchFieldDataType.Collection(SearchFieldDataType.Single),
                searchable=True,
                dimensions=1536,  # Set the dimensions explicitly
                vector_search_configuration="myHnswProfile",  # Link to the vector search profile
            ),
            SearchableField(name="metadata", type="Edm.String"),
            
            # Adding metadata fields as searchable fields
            SearchableField(name="date", type="Edm.String"),
            SearchableField(name="userid", type="Edm.String"),
            SearchableField(name="contractid", type="Edm.String"),
            SearchableField(name="title", type="Edm.String"),
            SearchableField(name="clauseid", type="Edm.String"),
            SearchableField(name="summary", type="Edm.String"),
            SearchableField(name="keywords", type="Edm.String"),
        ]

        vector_search = VectorSearch(
            algorithms=[
                HnswAlgorithmConfiguration(
                    name="myHnsw",
                    kind=VectorSearchAlgorithmKind.HNSW,
                    parameters={
                        "m": 4,
                        "efConstruction": 400,
                        "efSearch": 500,
                        "metric": VectorSearchAlgorithmMetric.COSINE
                    },
                ),
            ],
            profiles=[
                VectorSearchProfile(
                    name="myHnswProfile",
                    algorithm_configuration_name="myHnsw",
                ),
            ],
        )

        semantic_config = SemanticConfiguration(
            name="mySemanticConfig",
            prioritized_fields=SemanticPrioritizedFields(
                content_fields=[SemanticField(field_name="chunk")],
            ),
        )
        semantic_search = SemanticSearch(configurations=[semantic_config])

        index = SearchIndex(
            name=index_name,
            fields=fields,
            vector_search=vector_search,
            semantic_search=semantic_search,
        )

        # Create the index
        index_client.create_index(index)
        print(f"Index {index_name} created successfully.")

    except Exception as e:
        print(f"An error occurred: {e}")

# Run the function to create the index
create_search_index("your-index-name", index_client)

This code ensures that all specified fields, including "date", "userid", "contractid", "title", "clauseid", "summary", and "keywords", are searchable in the Azure Search index ^[1][2].

To continue talking to Dosu, mention @dosu.

[d3buggerdan]

@dosu
dimensions is not a known attribute of class <class 'azure.search.documents.indexes.models._index.SearchField'> and will be ignored
vector_search_configuration is not a known attribute of class <class 'azure.search.documents.indexes.models._index.SearchField'> and will be ignored
An error occurred: (InvalidRequestParameter) The request is invalid. Details: definition : The vector field 'embedding' must have the properties 'dimensions' and 'vectorSearchConfiguration' set.
Code: InvalidRequestParameter
Message: The request is invalid. Details: definition : The vector field 'embedding' must have the properties 'dimensions' and 'vectorSearchConfiguration' set.
Exception Details: (InvalidField) The vector field 'embedding' must have the properties 'dimensions' and 'vectorSearchConfiguration' set. Parameters: definition
Code: InvalidField
Message: The vector field 'embedding' must have the properties 'dimensions' and 'vectorSearchConfiguration' set. Parameters: definition

[dosubot[bot]]

To resolve the errors related to the dimensions and vector_search_configuration properties for the embedding field in your Azure Search index creation code, you need to ensure that the vector_search_dimensions and vector_search_profile_name properties are set appropriately. Here is the corrected code:

from azure.search.documents.indexes.models import (
    SimpleField, SearchableField, SearchIndex, VectorSearch, HnswAlgorithmConfiguration,
    VectorSearchAlgorithmKind, VectorSearchAlgorithmMetric, VectorSearchProfile, SemanticConfiguration,
    SemanticPrioritizedFields, SemanticField, SemanticSearch, SearchFieldDataType, SearchField
)

def create_search_index(index_name, index_client):
    try:
        fields = [
            SimpleField(name="id", type="Edm.String", key=True),
            SearchableField(name="chunk", type="Edm.String", analyzer_name="en.lucene"),
            SearchField(
                name="embedding",
                type=SearchFieldDataType.Collection(SearchFieldDataType.Single),
                searchable=True,
                vector_search_dimensions=1536,  # Set the dimensions explicitly
                vector_search_profile_name="myHnswProfile",  # Link to the vector search profile
            ),
            SearchableField(name="metadata", type="Edm.String"),
            
            # Adding metadata fields as searchable fields
            SearchableField(name="date", type="Edm.String"),
            SearchableField(name="userid", type="Edm.String"),
            SearchableField(name="contractid", type="Edm.String"),
            SearchableField(name="title", type="Edm.String"),
            SearchableField(name="clauseid", type="Edm.String"),
            SearchableField(name="summary", type="Edm.String"),
            SearchableField(name="keywords", type="Edm.String"),
        ]

        vector_search = VectorSearch(
            algorithms=[
                HnswAlgorithmConfiguration(
                    name="myHnsw",
                    kind=VectorSearchAlgorithmKind.HNSW,
                    parameters={
                        "m": 4,
                        "efConstruction": 400,
                        "efSearch": 500,
                        "metric": VectorSearchAlgorithmMetric.COSINE
                    },
                ),
            ],
            profiles=[
                VectorSearchProfile(
                    name="myHnswProfile",
                    algorithm_configuration_name="myHnsw",
                ),
            ],
        )

        semantic_config = SemanticConfiguration(
            name="mySemanticConfig",
            prioritized_fields=SemanticPrioritizedFields(
                content_fields=[SemanticField(field_name="chunk")],
            ),
        )
        semantic_search = SemanticSearch(configurations=[semantic_config])

        index = SearchIndex(
            name=index_name,
            fields=fields,
            vector_search=vector_search,
            semantic_search=semantic_search,
        )

        # Create the index
        index_client.create_index(index)
        print(f"Index {index_name} created successfully.")

    except Exception as e:
        print(f"An error occurred: {e}")

# Run the function to create the index
create_search_index("your-index-name", index_client)

In this code snippet, vector_search_dimensions is set to 1536, which should be the dimensionality of your embedding vectors. The vector_search_profile_name is set to "myHnswProfile", which should match the name of the vector search profile you have configured.

Make sure that you have configured the vector search algorithms and profiles as shown below:

vector_search = VectorSearch(
    algorithms=[
        HnswAlgorithmConfiguration(
            name="myHnsw",
            kind=VectorSearchAlgorithmKind.HNSW,
            parameters={
                "m": 4,
                "efConstruction": 400,
                "efSearch": 500,
                "metric": VectorSearchAlgorithmMetric.COSINE
            },
        ),
    ],
    profiles=[
        VectorSearchProfile(
            name="myHnswProfile",
            algorithm_configuration_name="myHnsw",
        ),
    ],
)

This should resolve the errors related to the dimensions and vector_search_configuration properties for the embedding field ^[1][2].

To continue talking to Dosu, mention @dosu.

[d3buggerdan]

@dosu Once i made the index, how can i store the embeddings in it this doesnt work?
from azure.search.documents.indexes.models import (
SimpleField, SearchableField, SearchIndex, VectorSearch, HnswAlgorithmConfiguration, VectorSearchAlgorithmKind,
VectorSearchAlgorithmMetric, VectorSearchProfile, SemanticConfiguration, SemanticPrioritizedFields, SemanticField,
SemanticSearch, SearchFieldDataType, SearchField
)

def create_search_index(index_name, index_client):
try:
fields = [
SimpleField(name="id", type="Edm.String", key=True, filterable=True, searchable=True, retrievable=True),
SearchableField(name="chunk", type="Edm.String", analyzer_name="en.lucene", filterable=True, searchable=True, retrievable=True),
SearchField(
name="embedding",
type=SearchFieldDataType.Collection(SearchFieldDataType.Single),
searchable=True,
filterable=False, # Typically, vector fields are not filterable
vector_search_dimensions=1536, # Set the dimensions explicitly
vector_search_profile_name="myHnswProfile", # Default to the HNSW vector search profile
),
SearchableField(name="metadata", type="Edm.String", filterable=True, searchable=True, retrievable=True),

        # Adding metadata fields as searchable fields
        SearchableField(name="doc_id", type="Edm.String", filterable=True, searchable=True, retrievable=True),
        SearchableField(name="date", type="Edm.String", filterable=True, searchable=True, retrievable=True),
        SearchableField(name="userid", type="Edm.String", filterable=True, searchable=True, retrievable=True),
        SearchableField(name="contractid", type="Edm.String", filterable=True, searchable=True, retrievable=True),
        SearchableField(name="title", type="Edm.String", filterable=True, searchable=True, retrievable=True),
        SearchableField(name="clauseid", type="Edm.String", filterable=True, searchable=True, retrievable=True),
        SearchableField(name="summary", type="Edm.String", filterable=True, searchable=True, retrievable=True),
        SearchableField(name="keywords", type="Edm.String", filterable=True, searchable=True, retrievable=True),
    ]

    vector_search = VectorSearch(
        algorithms=[
            HnswAlgorithmConfiguration(
                name="myHnsw",
                kind=VectorSearchAlgorithmKind.HNSW,
                parameters={
                    "m": 4,
                    "efConstruction": 400,
                    "efSearch": 500,
                    "metric": VectorSearchAlgorithmMetric.COSINE
                },
            ),
            HnswAlgorithmConfiguration(
                name="myExhaustiveKnn",
                kind=VectorSearchAlgorithmKind.EXHAUSTIVE_KNN,
                parameters={
                    "metric": VectorSearchAlgorithmMetric.COSINE
                },
            ),
        ],
        profiles=[
            VectorSearchProfile(
                name="myHnswProfile",
                algorithm_configuration_name="myHnsw",
            ),
            VectorSearchProfile(
                name="myExhaustiveKnnProfile",
                algorithm_configuration_name="myExhaustiveKnn",
            ),
        ],
    )

    semantic_config = SemanticConfiguration(
        name="mySemanticConfig",
        prioritized_fields=SemanticPrioritizedFields(
            content_fields=[SemanticField(field_name="chunk")],
        ),
    )
    semantic_search = SemanticSearch(configurations=[semantic_config])

    index = SearchIndex(
        name=index_name,
        fields=fields,
        vector_search=vector_search,
        semantic_search=semantic_search,
    )

    # Create the index
    index_client.create_index(index)
    print(f"Index {index_name} created successfully.")

except Exception as e:
    print(f"An error occurred: {e}")

Run the function to create the index

create_search_index(index_name, index_client)

"userid": ("userid", MetadataIndexFieldType.STRING),
"contractid": ("contractid", MetadataIndexFieldType.STRING),
"title": ("title", MetadataIndexFieldType.STRING),
"clauseid": ("clauseid", MetadataIndexFieldType.STRING),
"summary": ("summary", MetadataIndexFieldType.STRING),
"keywords": ("keywords", MetadataIndexFieldType.STRING)

metadata_fields = {
"date": ("date", MetadataIndexFieldType.STRING),
"userid": ("userid", MetadataIndexFieldType.STRING),
"contractid": ("contractid", MetadataIndexFieldType.STRING),
"title": ("title", MetadataIndexFieldType.STRING),
"clauseid": ("clauseid", MetadataIndexFieldType.STRING),
"summary": ("summary", MetadataIndexFieldType.STRING),
"keywords": ("keywords", MetadataIndexFieldType.STRING)
}

Define the vector store
vector_store = AzureAISearchVectorStore(
search_or_index_client=index_client,
filterable_metadata_field_keys=metadata_fields,
index_name=index_name,
index_management=IndexManagement.CREATE_IF_NOT_EXISTS,
id_field_key="id",
chunk_field_key="chunk",
embedding_field_key="embedding",
embedding_dimensionality=1536,
metadata_string_field_key="metadata",
doc_id_field_key="doc_id",
language_analyzer="en.lucene",
vector_algorithm_type="exhaustiveKnn",
)

Function to generate a summary for a clause using the LLM
def summarize_clause(llm, clause_text):
prompt = (
"prompt"
)
response = llm.complete(prompt)
summary = response.text.strip()
return summary

Function to generate keywords for a clause using the LLM
def generate_keywords(llm, clause_text):
prompt = (
"prompt"
)
response = llm.complete(prompt)

Process and clean up the keywords list

keywords = response.text.strip().split(', ')
keywords = [keyword.strip().lower() for keyword in keywords if len(keyword.split()) < 3] # Remove long phrases
return ', '.join(keywords[:10]) # Ensure the list is exactly 10 keywords, joined into a single string
Create document objects using the clauses from the contract
documents = []
for i, item in enumerate(contract_data["contract"]): # Fixed to use "document"
clause_text = item["clause"]
clause_id = f"clause_{i+1}" # Fixed clause ID to use a proper format

Generate a summary and keywords for the clause

clause_summary = summarize_clause(llm, clause_text)
clause_keywords = generate_keywords(llm, clause_text)

Create a document with the summary and keywords included in the metadata

documents.append(
Document(
text=clause_text,
id_=clause_id,
metadata={
"date": "<REPLACE_WITH_DATE>", # Replace with actual date
"userid": "<REPLACE_WITH_USERID>", # Replace with actual user ID
"contractid": "<REPLACE_WITH_CONTRACTID>", # Replace with actual contract ID
"title": "<REPLACE_WITH_TITLE>", # Replace with actual title
"clauseid": clause_id,
"summary": clause_summary, # Add the summary to the metadata
"keywords": clause_keywords # Add the keywords to the metadata
}
)
)
Create the vector store index with the summarized documents
storage_context = StorageContext.from_defaults(vector_store=vector_store)

Settings.llm = llm
Settings.embed_model = embed_model
index = VectorStoreIndex.from_documents(
documents, storage_context=storage_context
)

[dosubot[bot]]

To store the embeddings in the Azure Search index using the AzureAISearchVectorStore class, you need to ensure that the embeddings are correctly added to the index. Here is the complete code with the necessary updates:

1. Create the Azure Search Index:
   Ensure that the index is created with the correct fields and configurations.

2. Store the Embeddings:
   Use the add method of the AzureAISearchVectorStore class to add documents with embeddings to the index.

Here is the complete code:

from azure.search.documents.indexes.models import (
    SimpleField, SearchableField, SearchIndex, VectorSearch, HnswAlgorithmConfiguration,
    VectorSearchAlgorithmKind, VectorSearchAlgorithmMetric, VectorSearchProfile, SemanticConfiguration,
    SemanticPrioritizedFields, SemanticField, SemanticSearch, SearchFieldDataType, SearchField
)
from azure.core.credentials import AzureKeyCredential
from azure.search.documents.indexes import SearchIndexClient
from azure.search.documents import SearchClient
from llama_index.vector_stores.azureaisearch import AzureAISearchVectorStore, IndexManagement, MetadataIndexFieldType
from llama_index.core import VectorStoreIndex, StorageContext, Settings, Document

# Azure OpenAI setup
aoai_api_key = "YOUR_AZURE_OPENAI_API_KEY"
aoai_endpoint = "YOUR_AZURE_OPENAI_ENDPOINT"
aoai_api_version = "2023-05-15"

llm = AzureOpenAI(
    model="YOUR_AZURE_OPENAI_COMPLETION_MODEL_NAME",
    deployment_name="YOUR_AZURE_OPENAI_COMPLETION_DEPLOYMENT_NAME",
    api_key=aoai_api_key,
    azure_endpoint=aoai_endpoint,
    api_version=aoai_api_version,
)
embed_model = AzureOpenAIEmbedding(
    model="YOUR_AZURE_OPENAI_EMBEDDING_MODEL_NAME",
    deployment_name="YOUR_AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME",
    api_key=aoai_api_key,
    azure_endpoint=aoai_endpoint,
    api_version=aoai_api_version,
)

# Azure AI Search setup
search_service_api_key = "YOUR-AZURE-SEARCH-SERVICE-ADMIN-KEY"
search_service_endpoint = "YOUR-AZURE-SEARCH-SERVICE-ENDPOINT"
credential = AzureKeyCredential(search_service_api_key)

index_name = "llamaindex-vector-demo"

index_client = SearchIndexClient(
    endpoint=search_service_endpoint,
    credential=credential,
)

search_client = SearchClient(
    endpoint=search_service_endpoint,
    index_name=index_name,
    credential=credential,
)

metadata_fields = {
    "date": ("date", MetadataIndexFieldType.STRING),
    "userid": ("userid", MetadataIndexFieldType.STRING),
    "contractid": ("contractid", MetadataIndexFieldType.STRING),
    "title": ("title", MetadataIndexFieldType.STRING),
    "clauseid": ("clauseid", MetadataIndexFieldType.STRING),
    "summary": ("summary", MetadataIndexFieldType.STRING),
    "keywords": ("keywords", MetadataIndexFieldType.STRING),
}

# Define the vector store
vector_store = AzureAISearchVectorStore(
    search_or_index_client=index_client,
    filterable_metadata_field_keys=metadata_fields,
    index_name=index_name,
    index_management=IndexManagement.CREATE_IF_NOT_EXISTS,
    id_field_key="id",
    chunk_field_key="chunk",
    embedding_field_key="embedding",
    embedding_dimensionality=1536,
    metadata_string_field_key="metadata",
    doc_id_field_key="doc_id",
    language_analyzer="en.lucene",
    vector_algorithm_type="exhaustiveKnn",
)

# Function to generate a summary for a clause using the LLM
def summarize_clause(llm, clause_text):
    prompt = "prompt"
    response = llm.complete(prompt)
    summary = response.text.strip()
    return summary

# Function to generate keywords for a clause using the LLM
def generate_keywords(llm, clause_text):
    prompt = "prompt"
    response = llm.complete(prompt)
    
    # Process and clean up the keywords list
    keywords = response.text.strip().split(', ')
    keywords = [keyword.strip().lower() for keyword in keywords if len(keyword.split()) < 3]  # Remove long phrases
    return ', '.join(keywords[:10])  # Ensure the list is exactly 10 keywords, joined into a single string

# Create document objects using the clauses from the contract
documents = []
for i, item in enumerate(contract_data["contract"]):
    clause_text = item["clause"]
    clause_id = f"clause_{i+1}"
    
    # Generate a summary and keywords for the clause
    clause_summary = summarize_clause(llm, clause_text)
    clause_keywords = generate_keywords(llm, clause_text)
    
    # Create a document with the summary and keywords included in the metadata
    documents.append(
        Document(
            text=clause_text, 
            id_=clause_id, 
            metadata={
                "date": "<REPLACE_WITH_DATE>",
                "userid": "<REPLACE_WITH_USERID>",
                "contractid": "<REPLACE_WITH_CONTRACTID>",
                "title": "<REPLACE_WITH_TITLE>",
                "clauseid": clause_id,
                "summary": clause_summary,
                "keywords": clause_keywords
            }
        )
    )

# Create the vector store index with the summarized documents
storage_context = StorageContext.from_defaults(vector_store=vector_store)

Settings.llm = llm
Settings.embed_model = embed_model
index = VectorStoreIndex.from_documents(
    documents, storage_context=storage_context
)

# Add documents with embeddings to the index
vector_store.add(documents)

# Query the index
query_engine = index.as_query_engine(similarity_top_k=3)
response = query_engine.query("What did the author do growing up?")
print(response)

This code ensures that the embeddings are stored in the Azure Search index and can be queried effectively. The add method of the AzureAISearchVectorStore class is used to add the documents with embeddings to the index ^[1][2].

To continue talking to Dosu, mention @dosu.