From 35ee6bf8e48dc23606d0ac7a0238cca1cb6c04ff Mon Sep 17 00:00:00 2001 From: Roman Isecke <136338424+rbiseck3@users.noreply.github.com> Date: Wed, 17 Jul 2024 11:53:18 -0400 Subject: [PATCH] bugfix: conform all connectors to be added to registry (#3408) ### Description Looks like some connectors were never added to the registry explicitly since that change was introduced. All of them are now updated. --- .../ingest/v2/processes/connectors/__init__.py | 15 +++++++++++++++ .../ingest/v2/processes/connectors/mongodb.py | 16 ++++++---------- .../ingest/v2/processes/connectors/pinecone.py | 16 ++++++---------- .../ingest/v2/processes/connectors/sharepoint.py | 16 ++++++---------- .../v2/processes/connectors/singlestore.py | 16 ++++++---------- 5 files changed, 39 insertions(+), 40 deletions(-) diff --git a/unstructured/ingest/v2/processes/connectors/__init__.py b/unstructured/ingest/v2/processes/connectors/__init__.py index 81eb04189b..11a3d3be8d 100644 --- a/unstructured/ingest/v2/processes/connectors/__init__.py +++ b/unstructured/ingest/v2/processes/connectors/__init__.py @@ -18,12 +18,20 @@ from .google_drive import google_drive_source_entry from .local import CONNECTOR_TYPE as LOCAL_CONNECTOR_TYPE from .local import local_destination_entry, local_source_entry +from .mongodb import CONNECTOR_TYPE as MONGODB_CONNECTOR_TYPE +from .mongodb import mongodb_destination_entry from .onedrive import CONNECTOR_TYPE as ONEDRIVE_CONNECTOR_TYPE from .onedrive import onedrive_source_entry from .opensearch import CONNECTOR_TYPE as OPENSEARCH_CONNECTOR_TYPE from .opensearch import opensearch_destination_entry, opensearch_source_entry +from .pinecone import CONNECTOR_TYPE as PINECONE_CONNECTOR_TYPE +from .pinecone import pinecone_destination_entry from .salesforce import CONNECTOR_TYPE as SALESFORCE_CONNECTOR_TYPE from .salesforce import salesforce_source_entry +from .sharepoint import CONNECTOR_TYPE as SHAREPOINT_CONNECTOR_TYPE +from .sharepoint import sharepoint_source_entry +from .singlestore import CONNECTOR_TYPE as SINGLESTORE_CONNECTOR_TYPE +from .singlestore import singlestore_destination_entry from .sql import CONNECTOR_TYPE as SQL_CONNECTOR_TYPE from .sql import sql_destination_entry from .weaviate import CONNECTOR_TYPE as WEAVIATE_CONNECTOR_TYPE @@ -59,3 +67,10 @@ ) add_destination_entry(destination_type=SQL_CONNECTOR_TYPE, entry=sql_destination_entry) + +add_destination_entry(destination_type=MONGODB_CONNECTOR_TYPE, entry=mongodb_destination_entry) +add_destination_entry(destination_type=PINECONE_CONNECTOR_TYPE, entry=pinecone_destination_entry) +add_source_entry(source_type=SHAREPOINT_CONNECTOR_TYPE, entry=sharepoint_source_entry) +add_destination_entry( + destination_type=SINGLESTORE_CONNECTOR_TYPE, entry=singlestore_destination_entry +) diff --git a/unstructured/ingest/v2/processes/connectors/mongodb.py b/unstructured/ingest/v2/processes/connectors/mongodb.py index eb51a7bcc2..f5003911cd 100644 --- a/unstructured/ingest/v2/processes/connectors/mongodb.py +++ b/unstructured/ingest/v2/processes/connectors/mongodb.py @@ -19,7 +19,6 @@ from unstructured.ingest.v2.logger import logger from unstructured.ingest.v2.processes.connector_registry import ( DestinationRegistryEntry, - add_destination_entry, ) from unstructured.utils import requires_dependencies @@ -129,13 +128,10 @@ def run(self, contents: list[UploadContent], **kwargs: Any) -> None: collection.insert_many(chunk) -add_destination_entry( - destination_type=CONNECTOR_TYPE, - entry=DestinationRegistryEntry( - connection_config=MongoDBConnectionConfig, - uploader=MongoDBUploader, - uploader_config=MongoDBUploaderConfig, - upload_stager=MongoDBUploadStager, - upload_stager_config=MongoDBUploadStagerConfig, - ), +mongodb_destination_entry = DestinationRegistryEntry( + connection_config=MongoDBConnectionConfig, + uploader=MongoDBUploader, + uploader_config=MongoDBUploaderConfig, + upload_stager=MongoDBUploadStager, + upload_stager_config=MongoDBUploadStagerConfig, ) diff --git a/unstructured/ingest/v2/processes/connectors/pinecone.py b/unstructured/ingest/v2/processes/connectors/pinecone.py index 13c7527862..40648dd7f3 100644 --- a/unstructured/ingest/v2/processes/connectors/pinecone.py +++ b/unstructured/ingest/v2/processes/connectors/pinecone.py @@ -20,7 +20,6 @@ from unstructured.ingest.v2.logger import logger from unstructured.ingest.v2.processes.connector_registry import ( DestinationRegistryEntry, - add_destination_entry, ) from unstructured.staging.base import flatten_dict from unstructured.utils import requires_dependencies @@ -170,13 +169,10 @@ def run(self, contents: list[UploadContent], **kwargs: Any) -> None: ) -add_destination_entry( - destination_type=CONNECTOR_TYPE, - entry=DestinationRegistryEntry( - connection_config=PineconeConnectionConfig, - uploader=PineconeUploader, - uploader_config=PineconeUploaderConfig, - upload_stager=PineconeUploadStager, - upload_stager_config=PineconeUploadStagerConfig, - ), +pinecone_destination_entry = DestinationRegistryEntry( + connection_config=PineconeConnectionConfig, + uploader=PineconeUploader, + uploader_config=PineconeUploaderConfig, + upload_stager=PineconeUploadStager, + upload_stager_config=PineconeUploadStagerConfig, ) diff --git a/unstructured/ingest/v2/processes/connectors/sharepoint.py b/unstructured/ingest/v2/processes/connectors/sharepoint.py index dd766b4eef..696d327cec 100644 --- a/unstructured/ingest/v2/processes/connectors/sharepoint.py +++ b/unstructured/ingest/v2/processes/connectors/sharepoint.py @@ -24,7 +24,6 @@ from unstructured.ingest.v2.logger import logger from unstructured.ingest.v2.processes.connector_registry import ( SourceRegistryEntry, - add_source_entry, ) from unstructured.utils import requires_dependencies @@ -403,13 +402,10 @@ def run(self, file_data: FileData, **kwargs: Any) -> download_responses: return self.get_site_page(file_data=file_data) -add_source_entry( - source_type=CONNECTOR_TYPE, - entry=SourceRegistryEntry( - connection_config=SharepointConnectionConfig, - indexer_config=SharepointIndexerConfig, - indexer=SharepointIndexer, - downloader_config=SharepointDownloaderConfig, - downloader=SharepointDownloader, - ), +sharepoint_source_entry = SourceRegistryEntry( + connection_config=SharepointConnectionConfig, + indexer_config=SharepointIndexerConfig, + indexer=SharepointIndexer, + downloader_config=SharepointDownloaderConfig, + downloader=SharepointDownloader, ) diff --git a/unstructured/ingest/v2/processes/connectors/singlestore.py b/unstructured/ingest/v2/processes/connectors/singlestore.py index 3cdc5719b4..3e2d534e21 100644 --- a/unstructured/ingest/v2/processes/connectors/singlestore.py +++ b/unstructured/ingest/v2/processes/connectors/singlestore.py @@ -24,7 +24,6 @@ from unstructured.ingest.v2.logger import logger from unstructured.ingest.v2.processes.connector_registry import ( DestinationRegistryEntry, - add_destination_entry, ) from unstructured.utils import requires_dependencies @@ -152,13 +151,10 @@ def run(self, contents: list[UploadContent], **kwargs: Any) -> None: self.upload_csv(content=content) -add_destination_entry( - destination_type=CONNECTOR_TYPE, - entry=DestinationRegistryEntry( - connection_config=SingleStoreConnectionConfig, - uploader=SingleStoreUploader, - uploader_config=SingleStoreUploaderConfig, - upload_stager=SingleStoreUploadStager, - upload_stager_config=SingleStoreUploadStagerConfig, - ), +singlestore_destination_entry = DestinationRegistryEntry( + connection_config=SingleStoreConnectionConfig, + uploader=SingleStoreUploader, + uploader_config=SingleStoreUploaderConfig, + upload_stager=SingleStoreUploadStager, + upload_stager_config=SingleStoreUploadStagerConfig, )