From d070685dceeca2904b9e95443c42e6e566ae46af Mon Sep 17 00:00:00 2001 From: Roman Isecke Date: Tue, 16 Jul 2024 15:06:24 -0400 Subject: [PATCH] Conform all connectors to add to registry --- CHANGELOG.md | 2 +- unstructured/__version__.py | 2 +- .../ingest/v2/processes/connectors/__init__.py | 15 +++++++++++++++ .../ingest/v2/processes/connectors/mongodb.py | 16 ++++++---------- .../ingest/v2/processes/connectors/pinecone.py | 16 ++++++---------- .../ingest/v2/processes/connectors/sharepoint.py | 16 ++++++---------- .../v2/processes/connectors/singlestore.py | 16 ++++++---------- 7 files changed, 41 insertions(+), 42 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f38b81744c..bc8eb5aa21 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## 0.15.0-dev13 +## 0.15.0-dev14 ### Enhancements diff --git a/unstructured/__version__.py b/unstructured/__version__.py index 8bf8682774..1097befad5 100644 --- a/unstructured/__version__.py +++ b/unstructured/__version__.py @@ -1 +1 @@ -__version__ = "0.15.0-dev13" # pragma: no cover +__version__ = "0.15.0-dev14" # pragma: no cover diff --git a/unstructured/ingest/v2/processes/connectors/__init__.py b/unstructured/ingest/v2/processes/connectors/__init__.py index 81eb04189b..11a3d3be8d 100644 --- a/unstructured/ingest/v2/processes/connectors/__init__.py +++ b/unstructured/ingest/v2/processes/connectors/__init__.py @@ -18,12 +18,20 @@ from .google_drive import google_drive_source_entry from .local import CONNECTOR_TYPE as LOCAL_CONNECTOR_TYPE from .local import local_destination_entry, local_source_entry +from .mongodb import CONNECTOR_TYPE as MONGODB_CONNECTOR_TYPE +from .mongodb import mongodb_destination_entry from .onedrive import CONNECTOR_TYPE as ONEDRIVE_CONNECTOR_TYPE from .onedrive import onedrive_source_entry from .opensearch import CONNECTOR_TYPE as OPENSEARCH_CONNECTOR_TYPE from .opensearch import opensearch_destination_entry, opensearch_source_entry +from .pinecone import CONNECTOR_TYPE as PINECONE_CONNECTOR_TYPE +from .pinecone import pinecone_destination_entry from .salesforce import CONNECTOR_TYPE as SALESFORCE_CONNECTOR_TYPE from .salesforce import salesforce_source_entry +from .sharepoint import CONNECTOR_TYPE as SHAREPOINT_CONNECTOR_TYPE +from .sharepoint import sharepoint_source_entry +from .singlestore import CONNECTOR_TYPE as SINGLESTORE_CONNECTOR_TYPE +from .singlestore import singlestore_destination_entry from .sql import CONNECTOR_TYPE as SQL_CONNECTOR_TYPE from .sql import sql_destination_entry from .weaviate import CONNECTOR_TYPE as WEAVIATE_CONNECTOR_TYPE @@ -59,3 +67,10 @@ ) add_destination_entry(destination_type=SQL_CONNECTOR_TYPE, entry=sql_destination_entry) + +add_destination_entry(destination_type=MONGODB_CONNECTOR_TYPE, entry=mongodb_destination_entry) +add_destination_entry(destination_type=PINECONE_CONNECTOR_TYPE, entry=pinecone_destination_entry) +add_source_entry(source_type=SHAREPOINT_CONNECTOR_TYPE, entry=sharepoint_source_entry) +add_destination_entry( + destination_type=SINGLESTORE_CONNECTOR_TYPE, entry=singlestore_destination_entry +) diff --git a/unstructured/ingest/v2/processes/connectors/mongodb.py b/unstructured/ingest/v2/processes/connectors/mongodb.py index eb51a7bcc2..f5003911cd 100644 --- a/unstructured/ingest/v2/processes/connectors/mongodb.py +++ b/unstructured/ingest/v2/processes/connectors/mongodb.py @@ -19,7 +19,6 @@ from unstructured.ingest.v2.logger import logger from unstructured.ingest.v2.processes.connector_registry import ( DestinationRegistryEntry, - add_destination_entry, ) from unstructured.utils import requires_dependencies @@ -129,13 +128,10 @@ def run(self, contents: list[UploadContent], **kwargs: Any) -> None: collection.insert_many(chunk) -add_destination_entry( - destination_type=CONNECTOR_TYPE, - entry=DestinationRegistryEntry( - connection_config=MongoDBConnectionConfig, - uploader=MongoDBUploader, - uploader_config=MongoDBUploaderConfig, - upload_stager=MongoDBUploadStager, - upload_stager_config=MongoDBUploadStagerConfig, - ), +mongodb_destination_entry = DestinationRegistryEntry( + connection_config=MongoDBConnectionConfig, + uploader=MongoDBUploader, + uploader_config=MongoDBUploaderConfig, + upload_stager=MongoDBUploadStager, + upload_stager_config=MongoDBUploadStagerConfig, ) diff --git a/unstructured/ingest/v2/processes/connectors/pinecone.py b/unstructured/ingest/v2/processes/connectors/pinecone.py index 13c7527862..40648dd7f3 100644 --- a/unstructured/ingest/v2/processes/connectors/pinecone.py +++ b/unstructured/ingest/v2/processes/connectors/pinecone.py @@ -20,7 +20,6 @@ from unstructured.ingest.v2.logger import logger from unstructured.ingest.v2.processes.connector_registry import ( DestinationRegistryEntry, - add_destination_entry, ) from unstructured.staging.base import flatten_dict from unstructured.utils import requires_dependencies @@ -170,13 +169,10 @@ def run(self, contents: list[UploadContent], **kwargs: Any) -> None: ) -add_destination_entry( - destination_type=CONNECTOR_TYPE, - entry=DestinationRegistryEntry( - connection_config=PineconeConnectionConfig, - uploader=PineconeUploader, - uploader_config=PineconeUploaderConfig, - upload_stager=PineconeUploadStager, - upload_stager_config=PineconeUploadStagerConfig, - ), +pinecone_destination_entry = DestinationRegistryEntry( + connection_config=PineconeConnectionConfig, + uploader=PineconeUploader, + uploader_config=PineconeUploaderConfig, + upload_stager=PineconeUploadStager, + upload_stager_config=PineconeUploadStagerConfig, ) diff --git a/unstructured/ingest/v2/processes/connectors/sharepoint.py b/unstructured/ingest/v2/processes/connectors/sharepoint.py index dd766b4eef..696d327cec 100644 --- a/unstructured/ingest/v2/processes/connectors/sharepoint.py +++ b/unstructured/ingest/v2/processes/connectors/sharepoint.py @@ -24,7 +24,6 @@ from unstructured.ingest.v2.logger import logger from unstructured.ingest.v2.processes.connector_registry import ( SourceRegistryEntry, - add_source_entry, ) from unstructured.utils import requires_dependencies @@ -403,13 +402,10 @@ def run(self, file_data: FileData, **kwargs: Any) -> download_responses: return self.get_site_page(file_data=file_data) -add_source_entry( - source_type=CONNECTOR_TYPE, - entry=SourceRegistryEntry( - connection_config=SharepointConnectionConfig, - indexer_config=SharepointIndexerConfig, - indexer=SharepointIndexer, - downloader_config=SharepointDownloaderConfig, - downloader=SharepointDownloader, - ), +sharepoint_source_entry = SourceRegistryEntry( + connection_config=SharepointConnectionConfig, + indexer_config=SharepointIndexerConfig, + indexer=SharepointIndexer, + downloader_config=SharepointDownloaderConfig, + downloader=SharepointDownloader, ) diff --git a/unstructured/ingest/v2/processes/connectors/singlestore.py b/unstructured/ingest/v2/processes/connectors/singlestore.py index 3cdc5719b4..3e2d534e21 100644 --- a/unstructured/ingest/v2/processes/connectors/singlestore.py +++ b/unstructured/ingest/v2/processes/connectors/singlestore.py @@ -24,7 +24,6 @@ from unstructured.ingest.v2.logger import logger from unstructured.ingest.v2.processes.connector_registry import ( DestinationRegistryEntry, - add_destination_entry, ) from unstructured.utils import requires_dependencies @@ -152,13 +151,10 @@ def run(self, contents: list[UploadContent], **kwargs: Any) -> None: self.upload_csv(content=content) -add_destination_entry( - destination_type=CONNECTOR_TYPE, - entry=DestinationRegistryEntry( - connection_config=SingleStoreConnectionConfig, - uploader=SingleStoreUploader, - uploader_config=SingleStoreUploaderConfig, - upload_stager=SingleStoreUploadStager, - upload_stager_config=SingleStoreUploadStagerConfig, - ), +singlestore_destination_entry = DestinationRegistryEntry( + connection_config=SingleStoreConnectionConfig, + uploader=SingleStoreUploader, + uploader_config=SingleStoreUploaderConfig, + upload_stager=SingleStoreUploadStager, + upload_stager_config=SingleStoreUploadStagerConfig, )