rename microstrategy connector

open-metadata · Nov 12, 2024 · 739a5fd · 739a5fd
1 parent 8526687
commit 739a5fd
Show file tree

Hide file tree

Showing 24 changed files with 223 additions and 164 deletions.
diff --git a/bootstrap/sql/migrations/native/1.6.0/mysql/postDataMigrationSQLScript.sql b/bootstrap/sql/migrations/native/1.6.0/mysql/postDataMigrationSQLScript.sql
@@ -54,4 +54,24 @@ set json = JSON_SET(json, '$.connection.config.supportsSystemProfile', true)
 where serviceType in ('Snowflake', 'Redshift', 'BigQuery');
 
 -- Update all rows in the consumers_dlq table to set the source column to 'publisher'
-UPDATE consumers_dlq SET source = 'publisher';
+UPDATE consumers_dlq SET source = 'publisher';
+
+-- Update serviceType in dashboard_entity table
+UPDATE dashboard_entity
+SET json = JSON_SET(json, '$.serviceType', 'MicroStrategy')
+WHERE JSON_UNQUOTE(JSON_EXTRACT(json, '$.serviceType')) = 'Mstr';
+
+-- Update serviceType in dashboard_service_entity table
+UPDATE dashboard_service_entity
+SET json = JSON_SET(json, '$.serviceType', 'MicroStrategy')
+WHERE JSON_UNQUOTE(JSON_EXTRACT(json, '$.serviceType')) = 'Mstr';
+
+-- Update serviceType in dashboard_data_model_entity table
+UPDATE dashboard_data_model_entity
+SET json = JSON_SET(json, '$.serviceType', 'MicroStrategy')
+WHERE JSON_UNQUOTE(JSON_EXTRACT(json, '$.serviceType')) = 'Mstr';
+
+-- Update serviceType in chart_entity table
+UPDATE chart_entity
+SET json = JSON_SET(json, '$.serviceType', 'MicroStrategy')
+WHERE JSON_UNQUOTE(JSON_EXTRACT(json, '$.serviceType')) = 'Mstr';
diff --git a/bootstrap/sql/migrations/native/1.6.0/postgres/postDataMigrationSQLScript.sql b/bootstrap/sql/migrations/native/1.6.0/postgres/postDataMigrationSQLScript.sql
@@ -70,4 +70,24 @@ SET json = jsonb_set(json::jsonb, '{connection,config,supportsSystemProfile}', '
 WHERE serviceType IN ('Snowflake', 'Redshift', 'BigQuery');
 
 -- Update all rows in the consumers_dlq table to set the source column to 'publisher'
-UPDATE consumers_dlq SET source = 'publisher';
+UPDATE consumers_dlq SET source = 'publisher';
+
+-- Update serviceType in dashboard_entity table
+UPDATE dashboard_entity
+SET json = jsonb_set(json, '{serviceType}', '"MicroStrategy"')
+WHERE jsonb_extract_path_text(json, 'serviceType') = 'Mstr';
+
+-- Update serviceType in dashboard_service_entity table
+UPDATE dashboard_service_entity
+SET json = jsonb_set(json, '{serviceType}', '"MicroStrategy"')
+WHERE jsonb_extract_path_text(json, 'serviceType') = 'Mstr';
+
+-- Update serviceType in dashboard_data_model_entity table
+UPDATE dashboard_data_model_entity
+SET json = jsonb_set(json, '{serviceType}', '"MicroStrategy"')
+WHERE jsonb_extract_path_text(json, 'serviceType') = 'Mstr';
+
+-- Update serviceType in chart_entity table
+UPDATE chart_entity
+SET json = jsonb_set(json, '{serviceType}', '"MicroStrategy"')
+WHERE jsonb_extract_path_text(json, 'serviceType') = 'Mstr';
diff --git a/ingestion/setup.py b/ingestion/setup.py
@@ -304,7 +304,7 @@
         "psycopg2-binary",
         VERSIONS["geoalchemy2"],
     },
-    "mstr": {"mstr-rest-requests==0.14.1"},
+    "microstrategy": {"mstr-rest-requests==0.14.1"},
     "sagemaker": {VERSIONS["boto3"]},
     "salesforce": {"simple_salesforce~=1.11"},
     "sample-data": {VERSIONS["avro"], VERSIONS["grpc-tools"]},

diff --git a/...gestion/source/dashboard/mstr/__init__.py → ...ource/dashboard/microstrategy/__init__.py b/...gestion/source/dashboard/mstr/__init__.py → ...ource/dashboard/microstrategy/__init__.py
diff --git a/...ingestion/source/dashboard/mstr/client.py → .../source/dashboard/microstrategy/client.py b/...ingestion/source/dashboard/mstr/client.py → .../source/dashboard/microstrategy/client.py
@@ -9,26 +9,26 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 """
-REST Auth & Client for Mstr
+REST Auth & Client for MicroStrategy
 """
 import traceback
 from typing import List, Optional
 
 import requests
 from mstr.requests import MSTRRESTSession
 
-from metadata.generated.schema.entity.services.connections.dashboard.mstrConnection import (
-    MstrConnection,
+from metadata.generated.schema.entity.services.connections.dashboard.microStrategyConnection import (
+    MicroStrategyConnection,
 )
 from metadata.ingestion.connections.test_connections import SourceConnectionException
-from metadata.ingestion.source.dashboard.mstr.models import (
-    MstrDashboard,
-    MstrDashboardDetails,
-    MstrDashboardList,
-    MstrProject,
-    MstrProjectList,
-    MstrSearchResult,
-    MstrSearchResultList,
+from metadata.ingestion.source.dashboard.microstrategy.models import (
+    MicroStrategyDashboard,
+    MicroStrategyDashboardDetails,
+    MicroStrategyDashboardList,
+    MicroStrategyProject,
+    MicroStrategyProjectList,
+    MicroStrategySearchResult,
+    MicroStrategySearchResultList,
 )
 from metadata.utils.helpers import clean_uri
 from metadata.utils.logger import ingestion_logger
@@ -38,7 +38,7 @@
 API_VERSION = "MicroStrategyLibrary/api"
 
 
-class MSTRClient:
+class MicroStrategyClient:
     """
     Client Handling API communication with Metabase
     """
@@ -58,7 +58,7 @@ def _get_mstr_session(self) -> MSTRRESTSession:
             return session
 
         except KeyError as exe:
-            msg = "Failed to fetch mstr session, please validate credentials"
+            msg = "Failed to fetch MicroStrategy session, please validate credentials"
             raise SourceConnectionException(msg) from exe
 
         except Exception as exc:
@@ -67,15 +67,15 @@ def _get_mstr_session(self) -> MSTRRESTSession:
 
     def __init__(
         self,
-        config: MstrConnection,
+        config: MicroStrategyConnection,
     ):
         self.config = config
         self.session = self._get_mstr_session()
 
     def is_project_name(self) -> bool:
         return bool(self.config.projectName)
 
-    def get_projects_list(self) -> List[MstrProject]:
+    def get_projects_list(self) -> List[MicroStrategyProject]:
         """
         Get List of all projects
         """
@@ -87,7 +87,7 @@ def get_projects_list(self) -> List[MstrProject]:
             if not resp_projects.ok:
                 raise requests.ConnectionError()
 
-            project_list = MstrProjectList(projects=resp_projects.json())
+            project_list = MicroStrategyProjectList(projects=resp_projects.json())
             return project_list.projects
 
         except Exception as exc:
@@ -96,7 +96,7 @@ def get_projects_list(self) -> List[MstrProject]:
 
         return []
 
-    def get_project_by_name(self) -> Optional[MstrProject]:
+    def get_project_by_name(self) -> Optional[MicroStrategyProject]:
         """
         Get Project By Name
         """
@@ -109,7 +109,7 @@ def get_project_by_name(self) -> Optional[MstrProject]:
             if not resp_projects.ok:
                 raise requests.ConnectionError()
 
-            project = MstrProject(**resp_projects.json())
+            project = MicroStrategyProject(**resp_projects.json())
             return project
 
         except Exception:
@@ -120,7 +120,7 @@ def get_project_by_name(self) -> Optional[MstrProject]:
 
     def get_search_results_list(
         self, project_id, object_type
-    ) -> List[MstrSearchResult]:
+    ) -> List[MicroStrategySearchResult]:
         """
         Get Search Results
         """
@@ -147,7 +147,7 @@ def get_search_results_list(
             for resp_result in resp_results.json()["result"]:
                 results.append(resp_result)
 
-            results_list = MstrSearchResultList(results=results)
+            results_list = MicroStrategySearchResultList(results=results)
             return results_list.results
 
         except Exception:
@@ -156,7 +156,9 @@ def get_search_results_list(
 
         return []
 
-    def get_dashboards_list(self, project_id, project_name) -> List[MstrDashboard]:
+    def get_dashboards_list(
+        self, project_id, project_name
+    ) -> List[MicroStrategyDashboard]:
         """
         Get Dashboard
         """
@@ -168,10 +170,12 @@ def get_dashboards_list(self, project_id, project_name) -> List[MstrDashboard]:
             dashboards = []
             for result in results:
                 dashboards.append(
-                    MstrDashboard(projectName=project_name, **result.model_dump())
+                    MicroStrategyDashboard(
+                        projectName=project_name, **result.model_dump()
+                    )
                 )
 
-            dashboards_list = MstrDashboardList(dashboards=dashboards)
+            dashboards_list = MicroStrategyDashboardList(dashboards=dashboards)
             return dashboards_list.dashboards
 
         except Exception:
@@ -182,7 +186,7 @@ def get_dashboards_list(self, project_id, project_name) -> List[MstrDashboard]:
 
     def get_dashboard_details(
         self, project_id, project_name, dashboard_id
-    ) -> Optional[MstrDashboardDetails]:
+    ) -> Optional[MicroStrategyDashboardDetails]:
         """
         Get Dashboard Details
         """
@@ -198,7 +202,7 @@ def get_dashboard_details(
             if not resp_dashboard.ok:
                 raise requests.ConnectionError()
 
-            return MstrDashboardDetails(
+            return MicroStrategyDashboardDetails(
                 projectId=project_id, projectName=project_name, **resp_dashboard.json()
             )
 

diff --git a/...stion/source/dashboard/mstr/connection.py → ...rce/dashboard/microstrategy/connection.py b/...stion/source/dashboard/mstr/connection.py → ...rce/dashboard/microstrategy/connection.py
@@ -17,32 +17,27 @@
 from metadata.generated.schema.entity.automations.workflow import (
     Workflow as AutomationWorkflow,
 )
-from metadata.generated.schema.entity.services.connections.dashboard.mstrConnection import (
-    MstrConnection,
-)
-from metadata.generated.schema.entity.services.connections.testConnectionResult import (
-    TestConnectionResult,
+from metadata.generated.schema.entity.services.connections.dashboard.microStrategyConnection import (
+    MicroStrategyConnection,
 )
 from metadata.ingestion.connections.test_connections import test_connection_steps
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
-from metadata.ingestion.source.dashboard.mstr.client import MSTRClient
-from metadata.utils.constants import THREE_MIN
+from metadata.ingestion.source.dashboard.microstrategy.client import MicroStrategyClient
 
 
-def get_connection(connection: MstrConnection) -> MSTRClient:
+def get_connection(connection: MicroStrategyConnection) -> MicroStrategyClient:
     """
     Create connection
     """
-    return MSTRClient(connection)
+    return MicroStrategyClient(connection)
 
 
 def test_connection(
     metadata: OpenMetadata,
-    client: MSTRClient,
-    service_connection: MstrConnection,
+    client: MicroStrategyClient,
+    service_connection: MicroStrategyConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-    timeout_seconds: Optional[int] = THREE_MIN,
-) -> TestConnectionResult:
+) -> None:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
@@ -55,5 +50,4 @@ def test_connection(
         test_fn=test_fn,
         service_type=service_connection.type.value,
         automation_workflow=automation_workflow,
-        timeout_seconds=timeout_seconds,
     )
diff --git a/...gestion/source/dashboard/mstr/metadata.py → ...ource/dashboard/microstrategy/metadata.py b/...gestion/source/dashboard/mstr/metadata.py → ...ource/dashboard/microstrategy/metadata.py
@@ -8,16 +8,16 @@
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
-"""Mstr source module"""
+"""MicroStrategy source module"""
 import traceback
 from typing import Iterable, List, Optional
 
 from metadata.generated.schema.api.data.createChart import CreateChartRequest
 from metadata.generated.schema.api.data.createDashboard import CreateDashboardRequest
 from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
 from metadata.generated.schema.entity.data.chart import Chart
-from metadata.generated.schema.entity.services.connections.dashboard.mstrConnection import (
-    MstrConnection,
+from metadata.generated.schema.entity.services.connections.dashboard.microStrategyConnection import (
+    MicroStrategyConnection,
 )
 from metadata.generated.schema.entity.services.ingestionPipelines.status import (
     StackTraceError,
@@ -34,10 +34,10 @@
 from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.dashboard.dashboard_service import DashboardServiceSource
-from metadata.ingestion.source.dashboard.mstr.models import (
-    MstrDashboard,
-    MstrDashboardDetails,
-    MstrPage,
+from metadata.ingestion.source.dashboard.microstrategy.models import (
+    MicroStrategyDashboard,
+    MicroStrategyDashboardDetails,
+    MicroStrategyPage,
 )
 from metadata.utils import fqn
 from metadata.utils.filters import filter_by_chart
@@ -47,9 +47,9 @@
 logger = ingestion_logger()
 
 
-class MstrSource(DashboardServiceSource):
+class MicrostrategySource(DashboardServiceSource):
     """
-    MSTR Source Class
+    MicroStrategy Source Class
     """
 
     @classmethod
@@ -60,14 +60,14 @@ def create(
         pipeline_name: Optional[str] = None,
     ):
         config = WorkflowSource.model_validate(config_dict)
-        connection: MstrConnection = config.serviceConnection.root.config
-        if not isinstance(connection, MstrConnection):
+        connection: MicroStrategyConnection = config.serviceConnection.root.config
+        if not isinstance(connection, MicroStrategyConnection):
             raise InvalidSourceException(
-                f"Expected MstrConnection, but got {connection}"
+                f"Expected MicroStrategyConnection, but got {connection}"
             )
         return cls(config, metadata)
 
-    def get_dashboards_list(self) -> Optional[List[MstrDashboard]]:
+    def get_dashboards_list(self) -> Optional[List[MicroStrategyDashboard]]:
         """
         Get List of all dashboards
         """
@@ -85,13 +85,15 @@ def get_dashboards_list(self) -> Optional[List[MstrDashboard]]:
 
         return dashboards
 
-    def get_dashboard_name(self, dashboard: MstrDashboard) -> str:
+    def get_dashboard_name(self, dashboard: MicroStrategyDashboard) -> str:
         """
         Get Dashboard Name
         """
         return dashboard.name
 
-    def get_dashboard_details(self, dashboard: MstrDashboard) -> MstrDashboardDetails:
+    def get_dashboard_details(
+        self, dashboard: MicroStrategyDashboard
+    ) -> MicroStrategyDashboardDetails:
         """
         Get Dashboard Details
         """
@@ -101,7 +103,7 @@ def get_dashboard_details(self, dashboard: MstrDashboard) -> MstrDashboardDetail
         return dashboard_details
 
     def yield_dashboard(
-        self, dashboard_details: MstrDashboardDetails
+        self, dashboard_details: MicroStrategyDashboardDetails
     ) -> Iterable[Either[CreateDashboardRequest]]:
         """
         Method to Get Dashboard Entity
@@ -142,12 +144,12 @@ def yield_dashboard(
             )
 
     def yield_dashboard_lineage_details(
-        self, dashboard_details: MstrDashboardDetails, db_service_name: str
+        self, dashboard_details: MicroStrategyDashboardDetails, db_service_name: str
     ) -> Optional[Iterable[AddLineageRequest]]:
         """Not Implemented"""
 
     def yield_dashboard_chart(
-        self, dashboard_details: MstrDashboardDetails
+        self, dashboard_details: MicroStrategyDashboardDetails
     ) -> Optional[Iterable[CreateChartRequest]]:
         """Get chart method
 
@@ -166,7 +168,7 @@ def yield_dashboard_chart(
             logger.warning(f"Error creating dashboard: {exc}")
 
     def _yield_chart_from_visualization(
-        self, page: MstrPage
+        self, page: MicroStrategyPage
     ) -> Iterable[Either[CreateChartRequest]]:
         for chart in page.visualizations:
             try: