From 74e0371fa617d49da79a2fc3f12df2cfd890112d Mon Sep 17 00:00:00 2001 From: Jitendra Gundaniya Date: Fri, 3 May 2024 13:57:39 +0100 Subject: [PATCH 01/35] Add options for cli common with disable preview of datasets Signed-off-by: Jitendra Gundaniya --- package/kedro_viz/api/rest/requests.py | 1 + package/kedro_viz/api/rest/responses.py | 12 +++---- package/kedro_viz/api/rest/router.py | 2 +- .../integrations/deployment/aws_deployer.py | 4 ++- .../integrations/deployment/azure_deployer.py | 4 ++- .../integrations/deployment/base_deployer.py | 4 ++- .../deployment/deployer_factory.py | 10 +++--- .../integrations/deployment/gcp_deployer.py | 4 ++- .../integrations/deployment/local_deployer.py | 4 ++- package/kedro_viz/launchers/cli.py | 31 +++++++++++++++---- package/kedro_viz/models/flowchart.py | 19 ++++++++++++ package/kedro_viz/server.py | 3 +- 12 files changed, 74 insertions(+), 24 deletions(-) diff --git a/package/kedro_viz/api/rest/requests.py b/package/kedro_viz/api/rest/requests.py index 857ee12963..7a8c726e5e 100644 --- a/package/kedro_viz/api/rest/requests.py +++ b/package/kedro_viz/api/rest/requests.py @@ -7,5 +7,6 @@ class DeployerConfiguration(BaseModel): """Credentials for Deployers.""" platform: str + preview: bool endpoint: str bucket_name: str diff --git a/package/kedro_viz/api/rest/responses.py b/package/kedro_viz/api/rest/responses.py index 13aaa05a6d..d06aeb2c0d 100644 --- a/package/kedro_viz/api/rest/responses.py +++ b/package/kedro_viz/api/rest/responses.py @@ -320,7 +320,7 @@ def get_default_response() -> GraphAPIResponse: ) -def get_node_metadata_response(node_id: str): +def get_node_metadata_response(node_id: str, preview: bool = True): """API response for `/api/nodes/node_id`.""" node = data_access_manager.nodes.get_node_by_id(node_id) if not node: @@ -333,7 +333,7 @@ def get_node_metadata_response(node_id: str): return TaskNodeMetadata(task_node=node) if isinstance(node, DataNode): - return DataNodeMetadata(data_node=node) + return DataNodeMetadata(data_node=node, is_preview_enabled_for_all_nodes=preview) if isinstance(node, TranscodedDataNode): return TranscodedDataNodeMetadata(transcoded_data_node=node) @@ -419,12 +419,12 @@ def save_api_main_response_to_fs(main_path: str, remote_fs: Any): raise exc -def save_api_node_response_to_fs(nodes_path: str, remote_fs: Any): +def save_api_node_response_to_fs(nodes_path: str, remote_fs: Any, preview: bool): """Saves API /nodes/{node} response to a directory.""" for nodeId in data_access_manager.nodes.get_node_ids(): try: write_api_response_to_fs( - f"{nodes_path}/{nodeId}", get_node_metadata_response(nodeId), remote_fs + f"{nodes_path}/{nodeId}", get_node_metadata_response(nodeId, preview), remote_fs ) except Exception as exc: # pragma: no cover logger.exception( @@ -451,7 +451,7 @@ def save_api_pipeline_response_to_fs(pipelines_path: str, remote_fs: Any): raise exc -def save_api_responses_to_fs(path: str, remote_fs: Any): +def save_api_responses_to_fs(path: str, remote_fs: Any, preview: bool): """Saves all Kedro Viz API responses to a directory.""" try: logger.debug( @@ -469,7 +469,7 @@ def save_api_responses_to_fs(path: str, remote_fs: Any): remote_fs.makedirs(pipelines_path, exist_ok=True) save_api_main_response_to_fs(main_path, remote_fs) - save_api_node_response_to_fs(nodes_path, remote_fs) + save_api_node_response_to_fs(nodes_path, remote_fs, preview) save_api_pipeline_response_to_fs(pipelines_path, remote_fs) except Exception as exc: # pragma: no cover diff --git a/package/kedro_viz/api/rest/router.py b/package/kedro_viz/api/rest/router.py index 51d445c1a2..1b85d0d425 100644 --- a/package/kedro_viz/api/rest/router.py +++ b/package/kedro_viz/api/rest/router.py @@ -61,7 +61,7 @@ async def get_single_pipeline_data(registered_pipeline_id: str): async def deploy_kedro_viz(input_values: DeployerConfiguration): try: deployer = DeployerFactory.create_deployer( - input_values.platform, input_values.endpoint, input_values.bucket_name + input_values.platform, input_values.preview, input_values.endpoint, input_values.bucket_name ) deployer.deploy() response = { diff --git a/package/kedro_viz/integrations/deployment/aws_deployer.py b/package/kedro_viz/integrations/deployment/aws_deployer.py index 2a9f4636f4..4e1815283d 100644 --- a/package/kedro_viz/integrations/deployment/aws_deployer.py +++ b/package/kedro_viz/integrations/deployment/aws_deployer.py @@ -16,19 +16,21 @@ class AWSDeployer(BaseDeployer): """A class to handle the deployment of Kedro-viz to AWS S3. Attributes: + _preview (bool): Flag to indicate if dataset preview is enabled. _endpoint (str): AWS endpoint of the hosted site. _bucket_name (str): Name of the S3 bucket. _path (str): S3 protocol with bucket name. _fs (fsspec.filesystem): Filesystem for S3 protocol. """ - def __init__(self, endpoint, bucket_name): + def __init__(self, preview, endpoint, bucket_name): """Initialize S3Deployer with endpoint and bucket name. Args: endpoint (str): AWS endpoint of the hosted site. bucket_name (str): Name of the S3 bucket. """ super().__init__() + self._preview = preview self._endpoint = endpoint self._bucket_name = bucket_name self._path = f"{_S3_PROTOCOL}://{bucket_name}" diff --git a/package/kedro_viz/integrations/deployment/azure_deployer.py b/package/kedro_viz/integrations/deployment/azure_deployer.py index a147902545..0b135069b2 100644 --- a/package/kedro_viz/integrations/deployment/azure_deployer.py +++ b/package/kedro_viz/integrations/deployment/azure_deployer.py @@ -24,13 +24,14 @@ class AzureDeployer(BaseDeployer): """A class to handle the deployment of Kedro-viz to AzureBlobStorage. Attributes: + _preview (bool): Flag to indicate if dataset preview is enabled. _endpoint (str): Azure endpoint of the hosted site. _bucket_name (str): Name of the AzureBlobStorage account. _path (str): Container path for the AzureBlobStorage account. _fs (fsspec.filesystem): Filesystem for Azure protocol. """ - def __init__(self, endpoint, bucket_name): + def __init__(self, preview, endpoint, bucket_name): """Initialize AzureBlobStorage with endpoint and bucket name. Args: @@ -38,6 +39,7 @@ def __init__(self, endpoint, bucket_name): bucket_name (str): Name of the AzureBlobStorage account. """ super().__init__() + self._preview = preview self._endpoint = endpoint self._bucket_name = bucket_name self._path = f"{_AZ_PROTOCOL}://$web" diff --git a/package/kedro_viz/integrations/deployment/base_deployer.py b/package/kedro_viz/integrations/deployment/base_deployer.py index 22c4fcc819..4f9bf698a5 100644 --- a/package/kedro_viz/integrations/deployment/base_deployer.py +++ b/package/kedro_viz/integrations/deployment/base_deployer.py @@ -25,17 +25,19 @@ class BaseDeployer(abc.ABC): """A class to handle the creation of Kedro-viz build. Attributes: + _preview (bool): Flag to indicate if dataset preview is enabled. _path (str): build path name. _fs (fsspec.filesystem): Filesystem for local/remote protocol. """ def __init__(self): + self._preview = False self._path = None self._fs = None def _upload_api_responses(self): """Write API responses to the build.""" - save_api_responses_to_fs(self._path, self._fs) + save_api_responses_to_fs(self._path, self._fs, self._preview) def _ingest_heap_analytics(self): """Ingest heap analytics to index file in the build.""" diff --git a/package/kedro_viz/integrations/deployment/deployer_factory.py b/package/kedro_viz/integrations/deployment/deployer_factory.py index 5215bf8c00..eb331105c3 100644 --- a/package/kedro_viz/integrations/deployment/deployer_factory.py +++ b/package/kedro_viz/integrations/deployment/deployer_factory.py @@ -12,17 +12,17 @@ class DeployerFactory: """A class to handle creation of deployer class instances.""" @staticmethod - def create_deployer(platform, endpoint=None, bucket_name=None): + def create_deployer(platform, preview, endpoint=None, bucket_name=None): """Instantiate Kedro-viz deployer classes""" platform_name = platform.lower() if platform_name == "aws": - return AWSDeployer(endpoint, bucket_name) + return AWSDeployer(preview, endpoint, bucket_name) if platform_name == "azure": - return AzureDeployer(endpoint, bucket_name) + return AzureDeployer(preview, endpoint, bucket_name) if platform_name == "gcp": - return GCPDeployer(endpoint, bucket_name) + return GCPDeployer(preview, endpoint, bucket_name) if platform_name == "local": - return LocalDeployer() + return LocalDeployer(preview) raise ValueError( f"Invalid platform '{platform}' specified. \n" f"Kedro-Viz supports the following platforms - {*SHAREABLEVIZ_SUPPORTED_PLATFORMS,}" diff --git a/package/kedro_viz/integrations/deployment/gcp_deployer.py b/package/kedro_viz/integrations/deployment/gcp_deployer.py index c02010b24f..e9b9a3402e 100644 --- a/package/kedro_viz/integrations/deployment/gcp_deployer.py +++ b/package/kedro_viz/integrations/deployment/gcp_deployer.py @@ -18,13 +18,14 @@ class GCPDeployer(BaseDeployer): """A class to handle the deployment of Kedro-viz to Google Cloud Storage Bucket. Attributes: + _preview (bool): Flag to indicate if dataset preview is enabled. _endpoint (str): GCP endpoint of the hosted site. _bucket_name (str): Name of the GCP storage bucket. _path (str): GCP protocol with bucket name. _fs (fsspec.filesystem): Filesystem for GCP protocol. """ - def __init__(self, endpoint, bucket_name): + def __init__(self, preview, endpoint, bucket_name): """Initialize GCPDeployer with endpoint and bucket name. Args: @@ -32,6 +33,7 @@ def __init__(self, endpoint, bucket_name): bucket_name (str): Name of the GCP storage bucket. """ super().__init__() + self._preview = preview self._endpoint = endpoint self._bucket_name = bucket_name self._path = f"{_GCP_PROTOCOL}://{bucket_name}" diff --git a/package/kedro_viz/integrations/deployment/local_deployer.py b/package/kedro_viz/integrations/deployment/local_deployer.py index e3e5778521..a3cee9e2e6 100644 --- a/package/kedro_viz/integrations/deployment/local_deployer.py +++ b/package/kedro_viz/integrations/deployment/local_deployer.py @@ -19,12 +19,14 @@ class LocalDeployer(BaseDeployer): """A class to handle the creation of Kedro-viz build folder. Attributes: + _preview (bool): Flag to indicate if dataset preview is enabled. _build_path (str): build path name. _local_fs (fsspec.filesystem): Filesystem for local file protocol. """ - def __init__(self): + def __init__(self, preview): super().__init__() + self._preview = preview self._path = Path(_BUILD_PATH) self._path.mkdir(parents=True, exist_ok=True) self._fs = fsspec.filesystem(_FILE_PROTOCOL) diff --git a/package/kedro_viz/launchers/cli.py b/package/kedro_viz/launchers/cli.py index 5c2bcac75f..a9c6ef23b4 100644 --- a/package/kedro_viz/launchers/cli.py +++ b/package/kedro_viz/launchers/cli.py @@ -108,6 +108,11 @@ def viz(ctx): # pylint: disable=unused-argument is_flag=True, help="A flag to include all registered hooks in your Kedro Project", ) +@click.option( + "--preview", + default=True, + help="A flag to preview your node dataset" +) @click.option( "--params", type=click.UNPROCESSED, @@ -127,6 +132,7 @@ def run( autoreload, include_hooks, params, + preview, ): """Launch local Kedro Viz instance""" from kedro_viz.server import run_server @@ -170,6 +176,7 @@ def run( "include_hooks": include_hooks, "package_name": PACKAGE_NAME, "extra_params": params, + "preview": preview, } if autoreload: run_process_kwargs = { @@ -234,7 +241,12 @@ def run( is_flag=True, help="A flag to include all registered hooks in your Kedro Project", ) -def deploy(platform, endpoint, bucket_name, include_hooks): +@click.option( + "--preview", + default=False, + help="A flag to preview your node dataset", +) +def deploy(platform, endpoint, bucket_name, include_hooks, preview): """Deploy and host Kedro Viz on provided platform""" if not platform or platform.lower() not in SHAREABLEVIZ_SUPPORTED_PLATFORMS: display_cli_message( @@ -252,7 +264,7 @@ def deploy(platform, endpoint, bucket_name, include_hooks): ) return - create_shareableviz_process(platform, endpoint, bucket_name, include_hooks) + create_shareableviz_process(platform, preview, endpoint, bucket_name, include_hooks) @viz.command(context_settings={"help_option_names": ["-h", "--help"]}) @@ -261,14 +273,19 @@ def deploy(platform, endpoint, bucket_name, include_hooks): is_flag=True, help="A flag to include all registered hooks in your Kedro Project", ) -def build(include_hooks): +@click.option( + "--preview", + default=False, + help="A flag to preview your node dataset", +) +def build(include_hooks, preview): """Create build directory of local Kedro Viz instance with Kedro project data""" - create_shareableviz_process("local", include_hooks=include_hooks) + create_shareableviz_process("local", preview, include_hooks=include_hooks) def create_shareableviz_process( - platform, endpoint=None, bucket_name=None, include_hooks=False + platform, preview, endpoint=None, bucket_name=None, include_hooks=False ): """Creates platform specific deployer process""" try: @@ -279,6 +296,7 @@ def create_shareableviz_process( target=load_and_deploy_viz, args=( platform, + preview, endpoint, bucket_name, include_hooks, @@ -354,6 +372,7 @@ def create_shareableviz_process( def load_and_deploy_viz( platform, + preview, endpoint, bucket_name, include_hooks, @@ -368,7 +387,7 @@ def load_and_deploy_viz( ) # Start the deployment - deployer = DeployerFactory.create_deployer(platform, endpoint, bucket_name) + deployer = DeployerFactory.create_deployer(platform, preview, endpoint, bucket_name) deployer.deploy() except ( diff --git a/package/kedro_viz/models/flowchart.py b/package/kedro_viz/models/flowchart.py index e93418323c..308c836a5e 100644 --- a/package/kedro_viz/models/flowchart.py +++ b/package/kedro_viz/models/flowchart.py @@ -728,12 +728,14 @@ class DataNodeMetadata(GraphNodeMetadata): Args: data_node (DataNode): Data node to which this metadata belongs to. + is_preview_enabled_for_all_nodes (bool): Determines whether preview is enabled for all nodes Raises: AssertionError: If data_node is not supplied during instantiation """ data_node: DataNode = Field(..., exclude=True) + is_preview_enabled_for_all_nodes: bool = Field(..., exclude=True) type: Optional[str] = Field( default=None, validate_default=True, description="The type of the data node" @@ -769,6 +771,17 @@ class DataNodeMetadata(GraphNodeMetadata): description="The statistics for the data node.", ) + @model_validator(mode="before") + @classmethod + def check_is_preview_enabled_for_all_nodes_exists(cls, values): + assert "is_preview_enabled_for_all_nodes" in values + cls.set_is_preview_enabled_for_all_nodes(values["is_preview_enabled_for_all_nodes"]) + return values + + @classmethod + def set_is_preview_enabled_for_all_nodes(cls, is_preview_enabled_for_all_nodes): + cls.is_preview_enabled_for_all_nodes = is_preview_enabled_for_all_nodes + @model_validator(mode="before") @classmethod def check_data_node_exists(cls, values): @@ -806,6 +819,9 @@ def set_run_command(cls, _): @field_validator("preview") @classmethod def set_preview(cls, _): + if not cls.is_preview_enabled_for_all_nodes: + return None + if cls.data_node.is_preview_disabled() or not hasattr(cls.dataset, "preview"): return None @@ -829,6 +845,9 @@ def set_preview(cls, _): @field_validator("preview_type") @classmethod def set_preview_type(cls, _): + if not cls.is_preview_enabled_for_all_nodes: + return None + if cls.data_node.is_preview_disabled() or not hasattr(cls.dataset, "preview"): return None diff --git a/package/kedro_viz/server.py b/package/kedro_viz/server.py index eb31f7a9c9..cb76ab781f 100644 --- a/package/kedro_viz/server.py +++ b/package/kedro_viz/server.py @@ -90,6 +90,7 @@ def run_server( include_hooks: bool = False, package_name: Optional[str] = None, extra_params: Optional[Dict[str, Any]] = None, + preview: bool = True, ): # pylint: disable=redefined-outer-name """Run a uvicorn server with a FastAPI app that either launches API response data from a file or from reading data from a real Kedro project. @@ -127,7 +128,7 @@ def run_server( ) if save_file: - save_api_responses_to_fs(save_file, fsspec.filesystem("file")) + save_api_responses_to_fs(save_file, fsspec.filesystem("file"), preview) app = apps.create_api_app_from_project(path, autoreload) else: From 5398a1131cb9cf107ff2f92d9f04b624342d08d8 Mon Sep 17 00:00:00 2001 From: Jitendra Gundaniya Date: Tue, 7 May 2024 18:38:24 +0100 Subject: [PATCH 02/35] New approach for disable preview Signed-off-by: <> --- package/kedro_viz/api/rest/responses.py | 12 ++++---- package/kedro_viz/api/rest/router.py | 6 +++- package/kedro_viz/data_access/managers.py | 10 +++++++ .../integrations/deployment/aws_deployer.py | 4 +-- .../integrations/deployment/azure_deployer.py | 4 +-- .../integrations/deployment/base_deployer.py | 4 +-- .../deployment/deployer_factory.py | 10 +++---- .../integrations/deployment/gcp_deployer.py | 4 +-- .../integrations/deployment/local_deployer.py | 4 +-- package/kedro_viz/launchers/cli.py | 2 +- package/kedro_viz/models/flowchart.py | 29 ++++++------------- package/kedro_viz/server.py | 9 ++++-- 12 files changed, 47 insertions(+), 51 deletions(-) diff --git a/package/kedro_viz/api/rest/responses.py b/package/kedro_viz/api/rest/responses.py index d06aeb2c0d..13aaa05a6d 100644 --- a/package/kedro_viz/api/rest/responses.py +++ b/package/kedro_viz/api/rest/responses.py @@ -320,7 +320,7 @@ def get_default_response() -> GraphAPIResponse: ) -def get_node_metadata_response(node_id: str, preview: bool = True): +def get_node_metadata_response(node_id: str): """API response for `/api/nodes/node_id`.""" node = data_access_manager.nodes.get_node_by_id(node_id) if not node: @@ -333,7 +333,7 @@ def get_node_metadata_response(node_id: str, preview: bool = True): return TaskNodeMetadata(task_node=node) if isinstance(node, DataNode): - return DataNodeMetadata(data_node=node, is_preview_enabled_for_all_nodes=preview) + return DataNodeMetadata(data_node=node) if isinstance(node, TranscodedDataNode): return TranscodedDataNodeMetadata(transcoded_data_node=node) @@ -419,12 +419,12 @@ def save_api_main_response_to_fs(main_path: str, remote_fs: Any): raise exc -def save_api_node_response_to_fs(nodes_path: str, remote_fs: Any, preview: bool): +def save_api_node_response_to_fs(nodes_path: str, remote_fs: Any): """Saves API /nodes/{node} response to a directory.""" for nodeId in data_access_manager.nodes.get_node_ids(): try: write_api_response_to_fs( - f"{nodes_path}/{nodeId}", get_node_metadata_response(nodeId, preview), remote_fs + f"{nodes_path}/{nodeId}", get_node_metadata_response(nodeId), remote_fs ) except Exception as exc: # pragma: no cover logger.exception( @@ -451,7 +451,7 @@ def save_api_pipeline_response_to_fs(pipelines_path: str, remote_fs: Any): raise exc -def save_api_responses_to_fs(path: str, remote_fs: Any, preview: bool): +def save_api_responses_to_fs(path: str, remote_fs: Any): """Saves all Kedro Viz API responses to a directory.""" try: logger.debug( @@ -469,7 +469,7 @@ def save_api_responses_to_fs(path: str, remote_fs: Any, preview: bool): remote_fs.makedirs(pipelines_path, exist_ok=True) save_api_main_response_to_fs(main_path, remote_fs) - save_api_node_response_to_fs(nodes_path, remote_fs, preview) + save_api_node_response_to_fs(nodes_path, remote_fs) save_api_pipeline_response_to_fs(pipelines_path, remote_fs) except Exception as exc: # pragma: no cover diff --git a/package/kedro_viz/api/rest/router.py b/package/kedro_viz/api/rest/router.py index 1b85d0d425..82381f3e35 100644 --- a/package/kedro_viz/api/rest/router.py +++ b/package/kedro_viz/api/rest/router.py @@ -2,6 +2,7 @@ # pylint: disable=missing-function-docstring, broad-exception-caught import logging +from pathlib import Path from typing import List from fastapi import APIRouter @@ -10,6 +11,7 @@ from kedro_viz.api.rest.requests import DeployerConfiguration from kedro_viz.constants import PACKAGE_REQUIREMENTS from kedro_viz.integrations.deployment.deployer_factory import DeployerFactory +from package.kedro_viz.server import load_and_populate_data from .responses import ( APIErrorMessage, @@ -60,8 +62,10 @@ async def get_single_pipeline_data(registered_pipeline_id: str): @router.post("/deploy") async def deploy_kedro_viz(input_values: DeployerConfiguration): try: + load_and_populate_data(Path.cwd(), input_values.preview) + deployer = DeployerFactory.create_deployer( - input_values.platform, input_values.preview, input_values.endpoint, input_values.bucket_name + input_values.platform, input_values.endpoint, input_values.bucket_name ) deployer.deploy() response = { diff --git a/package/kedro_viz/data_access/managers.py b/package/kedro_viz/data_access/managers.py index 4e4e772e5a..4770db9108 100644 --- a/package/kedro_viz/data_access/managers.py +++ b/package/kedro_viz/data_access/managers.py @@ -64,6 +64,7 @@ def __init__(self): self.runs = RunsRepository() self.tracking_datasets = TrackingDatasetsRepository() self.dataset_stats = {} + self.is_preview_enabled_for_all_nodes = False def set_db_session(self, db_session_class: sessionmaker): """Set db session on repositories that need it.""" @@ -127,6 +128,14 @@ def add_dataset_stats(self, stats_dict: Dict): self.dataset_stats = stats_dict + def add_is_preview_enabled_for_all_nodes(self, is_preview_enabled_for_all_nodes: bool): + """Add the preview flag to the data access manager. + Args: + preview: A boolean flag to indicate whether the preview mode is enabled. + """ + + self.is_preview_enabled_for_all_nodes = is_preview_enabled_for_all_nodes + def get_stats_for_data_node(self, data_node_name: str) -> Union[Dict, None]: """Returns the dataset statistics for the data node if found @@ -304,6 +313,7 @@ def add_dataset( dataset=obj, stats=self.get_stats_for_data_node(_strip_transcoding(dataset_name)), is_free_input=is_free_input, + is_preview_enabled_for_all_nodes=self.is_preview_enabled_for_all_nodes, ) graph_node = self.nodes.add_node(graph_node) graph_node.add_pipeline(registered_pipeline_id) diff --git a/package/kedro_viz/integrations/deployment/aws_deployer.py b/package/kedro_viz/integrations/deployment/aws_deployer.py index 4e1815283d..2a9f4636f4 100644 --- a/package/kedro_viz/integrations/deployment/aws_deployer.py +++ b/package/kedro_viz/integrations/deployment/aws_deployer.py @@ -16,21 +16,19 @@ class AWSDeployer(BaseDeployer): """A class to handle the deployment of Kedro-viz to AWS S3. Attributes: - _preview (bool): Flag to indicate if dataset preview is enabled. _endpoint (str): AWS endpoint of the hosted site. _bucket_name (str): Name of the S3 bucket. _path (str): S3 protocol with bucket name. _fs (fsspec.filesystem): Filesystem for S3 protocol. """ - def __init__(self, preview, endpoint, bucket_name): + def __init__(self, endpoint, bucket_name): """Initialize S3Deployer with endpoint and bucket name. Args: endpoint (str): AWS endpoint of the hosted site. bucket_name (str): Name of the S3 bucket. """ super().__init__() - self._preview = preview self._endpoint = endpoint self._bucket_name = bucket_name self._path = f"{_S3_PROTOCOL}://{bucket_name}" diff --git a/package/kedro_viz/integrations/deployment/azure_deployer.py b/package/kedro_viz/integrations/deployment/azure_deployer.py index 0b135069b2..a147902545 100644 --- a/package/kedro_viz/integrations/deployment/azure_deployer.py +++ b/package/kedro_viz/integrations/deployment/azure_deployer.py @@ -24,14 +24,13 @@ class AzureDeployer(BaseDeployer): """A class to handle the deployment of Kedro-viz to AzureBlobStorage. Attributes: - _preview (bool): Flag to indicate if dataset preview is enabled. _endpoint (str): Azure endpoint of the hosted site. _bucket_name (str): Name of the AzureBlobStorage account. _path (str): Container path for the AzureBlobStorage account. _fs (fsspec.filesystem): Filesystem for Azure protocol. """ - def __init__(self, preview, endpoint, bucket_name): + def __init__(self, endpoint, bucket_name): """Initialize AzureBlobStorage with endpoint and bucket name. Args: @@ -39,7 +38,6 @@ def __init__(self, preview, endpoint, bucket_name): bucket_name (str): Name of the AzureBlobStorage account. """ super().__init__() - self._preview = preview self._endpoint = endpoint self._bucket_name = bucket_name self._path = f"{_AZ_PROTOCOL}://$web" diff --git a/package/kedro_viz/integrations/deployment/base_deployer.py b/package/kedro_viz/integrations/deployment/base_deployer.py index 4f9bf698a5..22c4fcc819 100644 --- a/package/kedro_viz/integrations/deployment/base_deployer.py +++ b/package/kedro_viz/integrations/deployment/base_deployer.py @@ -25,19 +25,17 @@ class BaseDeployer(abc.ABC): """A class to handle the creation of Kedro-viz build. Attributes: - _preview (bool): Flag to indicate if dataset preview is enabled. _path (str): build path name. _fs (fsspec.filesystem): Filesystem for local/remote protocol. """ def __init__(self): - self._preview = False self._path = None self._fs = None def _upload_api_responses(self): """Write API responses to the build.""" - save_api_responses_to_fs(self._path, self._fs, self._preview) + save_api_responses_to_fs(self._path, self._fs) def _ingest_heap_analytics(self): """Ingest heap analytics to index file in the build.""" diff --git a/package/kedro_viz/integrations/deployment/deployer_factory.py b/package/kedro_viz/integrations/deployment/deployer_factory.py index eb331105c3..5215bf8c00 100644 --- a/package/kedro_viz/integrations/deployment/deployer_factory.py +++ b/package/kedro_viz/integrations/deployment/deployer_factory.py @@ -12,17 +12,17 @@ class DeployerFactory: """A class to handle creation of deployer class instances.""" @staticmethod - def create_deployer(platform, preview, endpoint=None, bucket_name=None): + def create_deployer(platform, endpoint=None, bucket_name=None): """Instantiate Kedro-viz deployer classes""" platform_name = platform.lower() if platform_name == "aws": - return AWSDeployer(preview, endpoint, bucket_name) + return AWSDeployer(endpoint, bucket_name) if platform_name == "azure": - return AzureDeployer(preview, endpoint, bucket_name) + return AzureDeployer(endpoint, bucket_name) if platform_name == "gcp": - return GCPDeployer(preview, endpoint, bucket_name) + return GCPDeployer(endpoint, bucket_name) if platform_name == "local": - return LocalDeployer(preview) + return LocalDeployer() raise ValueError( f"Invalid platform '{platform}' specified. \n" f"Kedro-Viz supports the following platforms - {*SHAREABLEVIZ_SUPPORTED_PLATFORMS,}" diff --git a/package/kedro_viz/integrations/deployment/gcp_deployer.py b/package/kedro_viz/integrations/deployment/gcp_deployer.py index e9b9a3402e..c02010b24f 100644 --- a/package/kedro_viz/integrations/deployment/gcp_deployer.py +++ b/package/kedro_viz/integrations/deployment/gcp_deployer.py @@ -18,14 +18,13 @@ class GCPDeployer(BaseDeployer): """A class to handle the deployment of Kedro-viz to Google Cloud Storage Bucket. Attributes: - _preview (bool): Flag to indicate if dataset preview is enabled. _endpoint (str): GCP endpoint of the hosted site. _bucket_name (str): Name of the GCP storage bucket. _path (str): GCP protocol with bucket name. _fs (fsspec.filesystem): Filesystem for GCP protocol. """ - def __init__(self, preview, endpoint, bucket_name): + def __init__(self, endpoint, bucket_name): """Initialize GCPDeployer with endpoint and bucket name. Args: @@ -33,7 +32,6 @@ def __init__(self, preview, endpoint, bucket_name): bucket_name (str): Name of the GCP storage bucket. """ super().__init__() - self._preview = preview self._endpoint = endpoint self._bucket_name = bucket_name self._path = f"{_GCP_PROTOCOL}://{bucket_name}" diff --git a/package/kedro_viz/integrations/deployment/local_deployer.py b/package/kedro_viz/integrations/deployment/local_deployer.py index a3cee9e2e6..e3e5778521 100644 --- a/package/kedro_viz/integrations/deployment/local_deployer.py +++ b/package/kedro_viz/integrations/deployment/local_deployer.py @@ -19,14 +19,12 @@ class LocalDeployer(BaseDeployer): """A class to handle the creation of Kedro-viz build folder. Attributes: - _preview (bool): Flag to indicate if dataset preview is enabled. _build_path (str): build path name. _local_fs (fsspec.filesystem): Filesystem for local file protocol. """ - def __init__(self, preview): + def __init__(self): super().__init__() - self._preview = preview self._path = Path(_BUILD_PATH) self._path.mkdir(parents=True, exist_ok=True) self._fs = fsspec.filesystem(_FILE_PROTOCOL) diff --git a/package/kedro_viz/launchers/cli.py b/package/kedro_viz/launchers/cli.py index a9c6ef23b4..8ad959d9d8 100644 --- a/package/kedro_viz/launchers/cli.py +++ b/package/kedro_viz/launchers/cli.py @@ -383,7 +383,7 @@ def load_and_deploy_viz( """Loads Kedro Project data, creates a deployer and deploys to a platform""" try: load_and_populate_data( - Path.cwd(), include_hooks=include_hooks, package_name=package_name + Path.cwd(), is_preview_enabled_for_all_nodes=preview, include_hooks=include_hooks, package_name=package_name ) # Start the deployment diff --git a/package/kedro_viz/models/flowchart.py b/package/kedro_viz/models/flowchart.py index 308c836a5e..8d2a9cbc6b 100644 --- a/package/kedro_viz/models/flowchart.py +++ b/package/kedro_viz/models/flowchart.py @@ -228,6 +228,7 @@ def create_data_node( dataset: AbstractDataset, stats: Optional[Dict], is_free_input: bool = False, + is_preview_enabled_for_all_nodes: bool = False, ) -> Union["DataNode", "TranscodedDataNode"]: """Create a graph node of type data for a given Kedro Dataset instance. Args: @@ -263,6 +264,7 @@ def create_data_node( kedro_obj=dataset, is_free_input=is_free_input, stats=stats, + is_preview_enabled_for_all_nodes=is_preview_enabled_for_all_nodes, ) @classmethod @@ -576,6 +578,10 @@ class DataNode(GraphNode): ) stats: Optional[Dict] = Field(None, description="The statistics for the data node.") + is_preview_enabled_for_all_nodes: bool = Field( + None, description="The preview flag for all nodes" + ) + dataset_type: Optional[str] = Field( default=None, validate_default=True, @@ -648,7 +654,9 @@ def get_preview_args(self): def is_preview_disabled(self): """Checks if the dataset has a preview disabled""" return ( - self.viz_metadata is not None and self.viz_metadata.get("preview") is False + self.is_preview_enabled_for_all_nodes + or self.viz_metadata is not None + and self.viz_metadata.get("preview") is False ) @@ -728,14 +736,12 @@ class DataNodeMetadata(GraphNodeMetadata): Args: data_node (DataNode): Data node to which this metadata belongs to. - is_preview_enabled_for_all_nodes (bool): Determines whether preview is enabled for all nodes Raises: AssertionError: If data_node is not supplied during instantiation """ data_node: DataNode = Field(..., exclude=True) - is_preview_enabled_for_all_nodes: bool = Field(..., exclude=True) type: Optional[str] = Field( default=None, validate_default=True, description="The type of the data node" @@ -771,17 +777,6 @@ class DataNodeMetadata(GraphNodeMetadata): description="The statistics for the data node.", ) - @model_validator(mode="before") - @classmethod - def check_is_preview_enabled_for_all_nodes_exists(cls, values): - assert "is_preview_enabled_for_all_nodes" in values - cls.set_is_preview_enabled_for_all_nodes(values["is_preview_enabled_for_all_nodes"]) - return values - - @classmethod - def set_is_preview_enabled_for_all_nodes(cls, is_preview_enabled_for_all_nodes): - cls.is_preview_enabled_for_all_nodes = is_preview_enabled_for_all_nodes - @model_validator(mode="before") @classmethod def check_data_node_exists(cls, values): @@ -819,9 +814,6 @@ def set_run_command(cls, _): @field_validator("preview") @classmethod def set_preview(cls, _): - if not cls.is_preview_enabled_for_all_nodes: - return None - if cls.data_node.is_preview_disabled() or not hasattr(cls.dataset, "preview"): return None @@ -845,9 +837,6 @@ def set_preview(cls, _): @field_validator("preview_type") @classmethod def set_preview_type(cls, _): - if not cls.is_preview_enabled_for_all_nodes: - return None - if cls.data_node.is_preview_disabled() or not hasattr(cls.dataset, "preview"): return None diff --git a/package/kedro_viz/server.py b/package/kedro_viz/server.py index cb76ab781f..3130d0f56d 100644 --- a/package/kedro_viz/server.py +++ b/package/kedro_viz/server.py @@ -30,6 +30,7 @@ def populate_data( pipelines: Dict[str, Pipeline], session_store: BaseSessionStore, stats_dict: Dict, + is_preview_enabled_for_all_nodes: bool, ): # pylint: disable=redefined-outer-name """Populate data repositories. Should be called once on application start if creating an api app from project. @@ -48,9 +49,12 @@ def populate_data( data_access_manager.add_pipelines(pipelines) + data_access_manager.add_is_preview_enabled_for_all_nodes(is_preview_enabled_for_all_nodes) + def load_and_populate_data( path: Path, + is_preview_enabled_for_all_nodes: bool, env: Optional[str] = None, include_hooks: bool = False, package_name: Optional[str] = None, @@ -75,7 +79,7 @@ def load_and_populate_data( ) # Creates data repositories which are used by Kedro Viz Backend APIs - populate_data(data_access_manager, catalog, pipelines, session_store, stats_dict) + populate_data(data_access_manager, catalog, pipelines, session_store, stats_dict, is_preview_enabled_for_all_nodes) def run_server( @@ -90,7 +94,6 @@ def run_server( include_hooks: bool = False, package_name: Optional[str] = None, extra_params: Optional[Dict[str, Any]] = None, - preview: bool = True, ): # pylint: disable=redefined-outer-name """Run a uvicorn server with a FastAPI app that either launches API response data from a file or from reading data from a real Kedro project. @@ -128,7 +131,7 @@ def run_server( ) if save_file: - save_api_responses_to_fs(save_file, fsspec.filesystem("file"), preview) + save_api_responses_to_fs(save_file, fsspec.filesystem("file")) app = apps.create_api_app_from_project(path, autoreload) else: From 4da365f4f596aed92d4112dd8c625df326a1e901 Mon Sep 17 00:00:00 2001 From: Jitendra Gundaniya Date: Tue, 7 May 2024 20:19:05 +0100 Subject: [PATCH 03/35] Disable preview for cli and lint fix Signed-off-by: Jitendra Gundaniya --- package/kedro_viz/api/rest/router.py | 7 ++++--- package/kedro_viz/data_access/managers.py | 4 +++- package/kedro_viz/launchers/cli.py | 13 ++++++------- package/kedro_viz/models/flowchart.py | 2 +- package/kedro_viz/server.py | 17 ++++++++++++++--- 5 files changed, 28 insertions(+), 15 deletions(-) diff --git a/package/kedro_viz/api/rest/router.py b/package/kedro_viz/api/rest/router.py index 82381f3e35..15dad02039 100644 --- a/package/kedro_viz/api/rest/router.py +++ b/package/kedro_viz/api/rest/router.py @@ -11,7 +11,8 @@ from kedro_viz.api.rest.requests import DeployerConfiguration from kedro_viz.constants import PACKAGE_REQUIREMENTS from kedro_viz.integrations.deployment.deployer_factory import DeployerFactory -from package.kedro_viz.server import load_and_populate_data + +# from kedro_viz.server import load_and_populate_data from .responses import ( APIErrorMessage, @@ -62,8 +63,8 @@ async def get_single_pipeline_data(registered_pipeline_id: str): @router.post("/deploy") async def deploy_kedro_viz(input_values: DeployerConfiguration): try: - load_and_populate_data(Path.cwd(), input_values.preview) - + # load_and_populate_data(Path.cwd(), input_values.preview) + deployer = DeployerFactory.create_deployer( input_values.platform, input_values.endpoint, input_values.bucket_name ) diff --git a/package/kedro_viz/data_access/managers.py b/package/kedro_viz/data_access/managers.py index 4770db9108..74cdce5fb0 100644 --- a/package/kedro_viz/data_access/managers.py +++ b/package/kedro_viz/data_access/managers.py @@ -128,7 +128,9 @@ def add_dataset_stats(self, stats_dict: Dict): self.dataset_stats = stats_dict - def add_is_preview_enabled_for_all_nodes(self, is_preview_enabled_for_all_nodes: bool): + def add_is_preview_enabled_for_all_nodes( + self, is_preview_enabled_for_all_nodes: bool + ): """Add the preview flag to the data access manager. Args: preview: A boolean flag to indicate whether the preview mode is enabled. diff --git a/package/kedro_viz/launchers/cli.py b/package/kedro_viz/launchers/cli.py index 8ad959d9d8..d07add41c0 100644 --- a/package/kedro_viz/launchers/cli.py +++ b/package/kedro_viz/launchers/cli.py @@ -108,11 +108,6 @@ def viz(ctx): # pylint: disable=unused-argument is_flag=True, help="A flag to include all registered hooks in your Kedro Project", ) -@click.option( - "--preview", - default=True, - help="A flag to preview your node dataset" -) @click.option( "--params", type=click.UNPROCESSED, @@ -120,6 +115,7 @@ def viz(ctx): # pylint: disable=unused-argument help=PARAMS_ARG_HELP, callback=_split_params, ) +@click.option("--preview", default=True, help="A flag to preview your node dataset") # pylint: disable=import-outside-toplevel, too-many-locals def run( host, @@ -383,11 +379,14 @@ def load_and_deploy_viz( """Loads Kedro Project data, creates a deployer and deploys to a platform""" try: load_and_populate_data( - Path.cwd(), is_preview_enabled_for_all_nodes=preview, include_hooks=include_hooks, package_name=package_name + Path.cwd(), + is_preview_enabled_for_all_nodes=preview, + include_hooks=include_hooks, + package_name=package_name, ) # Start the deployment - deployer = DeployerFactory.create_deployer(platform, preview, endpoint, bucket_name) + deployer = DeployerFactory.create_deployer(platform, endpoint, bucket_name) deployer.deploy() except ( diff --git a/package/kedro_viz/models/flowchart.py b/package/kedro_viz/models/flowchart.py index 8d2a9cbc6b..f49a7fd378 100644 --- a/package/kedro_viz/models/flowchart.py +++ b/package/kedro_viz/models/flowchart.py @@ -654,7 +654,7 @@ def get_preview_args(self): def is_preview_disabled(self): """Checks if the dataset has a preview disabled""" return ( - self.is_preview_enabled_for_all_nodes + self.is_preview_enabled_for_all_nodes is False or self.viz_metadata is not None and self.viz_metadata.get("preview") is False ) diff --git a/package/kedro_viz/server.py b/package/kedro_viz/server.py index 3130d0f56d..f3168e6c8e 100644 --- a/package/kedro_viz/server.py +++ b/package/kedro_viz/server.py @@ -41,6 +41,10 @@ def populate_data( session_class = make_db_session_factory(session_store.location) data_access_manager.set_db_session(session_class) + data_access_manager.add_is_preview_enabled_for_all_nodes( + is_preview_enabled_for_all_nodes + ) + data_access_manager.add_catalog(catalog, pipelines) # add dataset stats before adding pipelines as the data nodes @@ -49,8 +53,6 @@ def populate_data( data_access_manager.add_pipelines(pipelines) - data_access_manager.add_is_preview_enabled_for_all_nodes(is_preview_enabled_for_all_nodes) - def load_and_populate_data( path: Path, @@ -79,12 +81,20 @@ def load_and_populate_data( ) # Creates data repositories which are used by Kedro Viz Backend APIs - populate_data(data_access_manager, catalog, pipelines, session_store, stats_dict, is_preview_enabled_for_all_nodes) + populate_data( + data_access_manager, + catalog, + pipelines, + session_store, + stats_dict, + is_preview_enabled_for_all_nodes, + ) def run_server( host: str = DEFAULT_HOST, port: int = DEFAULT_PORT, + preview: bool = True, load_file: Optional[str] = None, save_file: Optional[str] = None, pipeline_name: Optional[str] = None, @@ -123,6 +133,7 @@ def run_server( if load_file is None: load_and_populate_data( path, + preview, env, include_hooks, package_name, From 9f2afa257b5078629029359d25864900af6306cb Mon Sep 17 00:00:00 2001 From: Jitendra Gundaniya Date: Wed, 8 May 2024 11:34:59 +0100 Subject: [PATCH 04/35] deploy api with preview Signed-off-by: Jitendra Gundaniya --- package/kedro_viz/api/rest/router.py | 5 ++++- src/components/shareable-url-modal/shareable-url-modal.js | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/package/kedro_viz/api/rest/router.py b/package/kedro_viz/api/rest/router.py index 15dad02039..c320ff374b 100644 --- a/package/kedro_viz/api/rest/router.py +++ b/package/kedro_viz/api/rest/router.py @@ -62,8 +62,11 @@ async def get_single_pipeline_data(registered_pipeline_id: str): @router.post("/deploy") async def deploy_kedro_viz(input_values: DeployerConfiguration): + # Importing here to avoid circular imports + from kedro_viz.server import load_and_populate_data # pylint: disable=import-outside-toplevel + try: - # load_and_populate_data(Path.cwd(), input_values.preview) + load_and_populate_data(Path.cwd(), input_values.preview) deployer = DeployerFactory.create_deployer( input_values.platform, input_values.endpoint, input_values.bucket_name diff --git a/src/components/shareable-url-modal/shareable-url-modal.js b/src/components/shareable-url-modal/shareable-url-modal.js index 9d58bee238..58c582d4e9 100644 --- a/src/components/shareable-url-modal/shareable-url-modal.js +++ b/src/components/shareable-url-modal/shareable-url-modal.js @@ -104,7 +104,7 @@ const ShareableUrlModal = ({ onToggleModal, visible }) => { 'Content-Type': 'application/json', }, method: 'POST', - body: JSON.stringify(inputValues), + body: JSON.stringify({ ...inputValues, preview: false }), // preview appended to the body to disable the preview data }); const response = await request.json(); From 35f14367a4c1e2c232172541550fe42762671a9c Mon Sep 17 00:00:00 2001 From: Jitendra Gundaniya Date: Wed, 8 May 2024 14:45:10 +0100 Subject: [PATCH 05/35] Removing UI changes Signed-off-by: Jitendra Gundaniya --- package/kedro_viz/api/rest/router.py | 8 -------- package/kedro_viz/data_access/managers.py | 2 +- package/kedro_viz/launchers/cli.py | 10 +++++++--- package/kedro_viz/models/flowchart.py | 6 ++++-- .../shareable-url-modal/shareable-url-modal.js | 2 +- 5 files changed, 13 insertions(+), 15 deletions(-) diff --git a/package/kedro_viz/api/rest/router.py b/package/kedro_viz/api/rest/router.py index c320ff374b..51d445c1a2 100644 --- a/package/kedro_viz/api/rest/router.py +++ b/package/kedro_viz/api/rest/router.py @@ -2,7 +2,6 @@ # pylint: disable=missing-function-docstring, broad-exception-caught import logging -from pathlib import Path from typing import List from fastapi import APIRouter @@ -12,8 +11,6 @@ from kedro_viz.constants import PACKAGE_REQUIREMENTS from kedro_viz.integrations.deployment.deployer_factory import DeployerFactory -# from kedro_viz.server import load_and_populate_data - from .responses import ( APIErrorMessage, GraphAPIResponse, @@ -62,12 +59,7 @@ async def get_single_pipeline_data(registered_pipeline_id: str): @router.post("/deploy") async def deploy_kedro_viz(input_values: DeployerConfiguration): - # Importing here to avoid circular imports - from kedro_viz.server import load_and_populate_data # pylint: disable=import-outside-toplevel - try: - load_and_populate_data(Path.cwd(), input_values.preview) - deployer = DeployerFactory.create_deployer( input_values.platform, input_values.endpoint, input_values.bucket_name ) diff --git a/package/kedro_viz/data_access/managers.py b/package/kedro_viz/data_access/managers.py index 74cdce5fb0..5ec03035cf 100644 --- a/package/kedro_viz/data_access/managers.py +++ b/package/kedro_viz/data_access/managers.py @@ -133,7 +133,7 @@ def add_is_preview_enabled_for_all_nodes( ): """Add the preview flag to the data access manager. Args: - preview: A boolean flag to indicate whether the preview mode is enabled. + is_preview_enabled_for_all_nodes: A boolean flag to indicate whether the preview is enabled for all the nodes. """ self.is_preview_enabled_for_all_nodes = is_preview_enabled_for_all_nodes diff --git a/package/kedro_viz/launchers/cli.py b/package/kedro_viz/launchers/cli.py index d07add41c0..47b64ab586 100644 --- a/package/kedro_viz/launchers/cli.py +++ b/package/kedro_viz/launchers/cli.py @@ -115,7 +115,11 @@ def viz(ctx): # pylint: disable=unused-argument help=PARAMS_ARG_HELP, callback=_split_params, ) -@click.option("--preview", default=True, help="A flag to preview your node dataset") +@click.option( + "--preview", + default=True, + help="A flag to enable/disable a quick preview of node datasets.", +) # pylint: disable=import-outside-toplevel, too-many-locals def run( host, @@ -240,7 +244,7 @@ def run( @click.option( "--preview", default=False, - help="A flag to preview your node dataset", + help="A flag to enable/disable a quick preview of node datasets.", ) def deploy(platform, endpoint, bucket_name, include_hooks, preview): """Deploy and host Kedro Viz on provided platform""" @@ -272,7 +276,7 @@ def deploy(platform, endpoint, bucket_name, include_hooks, preview): @click.option( "--preview", default=False, - help="A flag to preview your node dataset", + help="A flag to enable/disable a quick preview of node datasets.", ) def build(include_hooks, preview): """Create build directory of local Kedro Viz instance with Kedro project data""" diff --git a/package/kedro_viz/models/flowchart.py b/package/kedro_viz/models/flowchart.py index f49a7fd378..8b228e4f72 100644 --- a/package/kedro_viz/models/flowchart.py +++ b/package/kedro_viz/models/flowchart.py @@ -227,8 +227,8 @@ def create_data_node( tags: Set[str], dataset: AbstractDataset, stats: Optional[Dict], + is_preview_enabled_for_all_nodes: bool, is_free_input: bool = False, - is_preview_enabled_for_all_nodes: bool = False, ) -> Union["DataNode", "TranscodedDataNode"]: """Create a graph node of type data for a given Kedro Dataset instance. Args: @@ -240,6 +240,7 @@ def create_data_node( dataset: A dataset in a Kedro pipeline. stats: The dictionary of dataset statistics, e.g. {"rows":2, "columns":3, "file_size":100} + is_preview_enabled_for_all_nodes: A flag to enable/disable a quick preview of node datasets. is_free_input: Whether the dataset is a free input in the pipeline Returns: An instance of DataNode. @@ -565,6 +566,7 @@ class DataNode(GraphNode): layer (Optional[str]): The layer that this data node belongs to. Defaults to `None`. is_free_input (bool): Determines whether the data node is a free input. Defaults to `False`. stats (Optional[Dict]): Statistics for the data node. Defaults to `None`. + is_preview_enabled_for_all_nodes (bool): A flag to enable/disable a quick preview of node datasets. Defaults to `False`. Raises: AssertionError: If kedro_obj, name are not supplied during instantiation @@ -579,7 +581,7 @@ class DataNode(GraphNode): stats: Optional[Dict] = Field(None, description="The statistics for the data node.") is_preview_enabled_for_all_nodes: bool = Field( - None, description="The preview flag for all nodes" + False, description="A flag to enable/disable a quick preview of node datasets." ) dataset_type: Optional[str] = Field( diff --git a/src/components/shareable-url-modal/shareable-url-modal.js b/src/components/shareable-url-modal/shareable-url-modal.js index 58c582d4e9..9d58bee238 100644 --- a/src/components/shareable-url-modal/shareable-url-modal.js +++ b/src/components/shareable-url-modal/shareable-url-modal.js @@ -104,7 +104,7 @@ const ShareableUrlModal = ({ onToggleModal, visible }) => { 'Content-Type': 'application/json', }, method: 'POST', - body: JSON.stringify({ ...inputValues, preview: false }), // preview appended to the body to disable the preview data + body: JSON.stringify(inputValues), }); const response = await request.json(); From 85e1a2da211ffdd20ee6c2d0c488b5ee45c3a9f9 Mon Sep 17 00:00:00 2001 From: Jitendra Gundaniya Date: Wed, 8 May 2024 14:47:14 +0100 Subject: [PATCH 06/35] Removing UI changes Signed-off-by: Jitendra Gundaniya --- package/kedro_viz/api/rest/requests.py | 1 - 1 file changed, 1 deletion(-) diff --git a/package/kedro_viz/api/rest/requests.py b/package/kedro_viz/api/rest/requests.py index 7a8c726e5e..857ee12963 100644 --- a/package/kedro_viz/api/rest/requests.py +++ b/package/kedro_viz/api/rest/requests.py @@ -7,6 +7,5 @@ class DeployerConfiguration(BaseModel): """Credentials for Deployers.""" platform: str - preview: bool endpoint: str bucket_name: str From f8f96cfe65aa19d727438893d754cb9012287bfd Mon Sep 17 00:00:00 2001 From: Jitendra Gundaniya Date: Wed, 8 May 2024 17:19:03 +0100 Subject: [PATCH 07/35] Disable preview from UI Signed-off-by: <> --- package/kedro_viz/api/rest/requests.py | 1 + package/kedro_viz/api/rest/responses.py | 12 ++++++------ package/kedro_viz/api/rest/router.py | 2 +- .../integrations/deployment/aws_deployer.py | 3 ++- .../integrations/deployment/azure_deployer.py | 3 ++- .../integrations/deployment/base_deployer.py | 3 ++- .../integrations/deployment/deployer_factory.py | 8 ++++---- package/kedro_viz/models/flowchart.py | 17 +++++++++++++++++ .../shareable-url-modal/shareable-url-modal.js | 2 +- 9 files changed, 36 insertions(+), 15 deletions(-) diff --git a/package/kedro_viz/api/rest/requests.py b/package/kedro_viz/api/rest/requests.py index 857ee12963..7a8c726e5e 100644 --- a/package/kedro_viz/api/rest/requests.py +++ b/package/kedro_viz/api/rest/requests.py @@ -7,5 +7,6 @@ class DeployerConfiguration(BaseModel): """Credentials for Deployers.""" platform: str + preview: bool endpoint: str bucket_name: str diff --git a/package/kedro_viz/api/rest/responses.py b/package/kedro_viz/api/rest/responses.py index 13aaa05a6d..d06aeb2c0d 100644 --- a/package/kedro_viz/api/rest/responses.py +++ b/package/kedro_viz/api/rest/responses.py @@ -320,7 +320,7 @@ def get_default_response() -> GraphAPIResponse: ) -def get_node_metadata_response(node_id: str): +def get_node_metadata_response(node_id: str, preview: bool = True): """API response for `/api/nodes/node_id`.""" node = data_access_manager.nodes.get_node_by_id(node_id) if not node: @@ -333,7 +333,7 @@ def get_node_metadata_response(node_id: str): return TaskNodeMetadata(task_node=node) if isinstance(node, DataNode): - return DataNodeMetadata(data_node=node) + return DataNodeMetadata(data_node=node, is_preview_enabled_for_all_nodes=preview) if isinstance(node, TranscodedDataNode): return TranscodedDataNodeMetadata(transcoded_data_node=node) @@ -419,12 +419,12 @@ def save_api_main_response_to_fs(main_path: str, remote_fs: Any): raise exc -def save_api_node_response_to_fs(nodes_path: str, remote_fs: Any): +def save_api_node_response_to_fs(nodes_path: str, remote_fs: Any, preview: bool): """Saves API /nodes/{node} response to a directory.""" for nodeId in data_access_manager.nodes.get_node_ids(): try: write_api_response_to_fs( - f"{nodes_path}/{nodeId}", get_node_metadata_response(nodeId), remote_fs + f"{nodes_path}/{nodeId}", get_node_metadata_response(nodeId, preview), remote_fs ) except Exception as exc: # pragma: no cover logger.exception( @@ -451,7 +451,7 @@ def save_api_pipeline_response_to_fs(pipelines_path: str, remote_fs: Any): raise exc -def save_api_responses_to_fs(path: str, remote_fs: Any): +def save_api_responses_to_fs(path: str, remote_fs: Any, preview: bool): """Saves all Kedro Viz API responses to a directory.""" try: logger.debug( @@ -469,7 +469,7 @@ def save_api_responses_to_fs(path: str, remote_fs: Any): remote_fs.makedirs(pipelines_path, exist_ok=True) save_api_main_response_to_fs(main_path, remote_fs) - save_api_node_response_to_fs(nodes_path, remote_fs) + save_api_node_response_to_fs(nodes_path, remote_fs, preview) save_api_pipeline_response_to_fs(pipelines_path, remote_fs) except Exception as exc: # pragma: no cover diff --git a/package/kedro_viz/api/rest/router.py b/package/kedro_viz/api/rest/router.py index 51d445c1a2..7b381d02bc 100644 --- a/package/kedro_viz/api/rest/router.py +++ b/package/kedro_viz/api/rest/router.py @@ -61,7 +61,7 @@ async def get_single_pipeline_data(registered_pipeline_id: str): async def deploy_kedro_viz(input_values: DeployerConfiguration): try: deployer = DeployerFactory.create_deployer( - input_values.platform, input_values.endpoint, input_values.bucket_name + input_values.platform, input_values.preview,input_values.endpoint, input_values.bucket_name ) deployer.deploy() response = { diff --git a/package/kedro_viz/integrations/deployment/aws_deployer.py b/package/kedro_viz/integrations/deployment/aws_deployer.py index 2a9f4636f4..93c27aeba5 100644 --- a/package/kedro_viz/integrations/deployment/aws_deployer.py +++ b/package/kedro_viz/integrations/deployment/aws_deployer.py @@ -22,13 +22,14 @@ class AWSDeployer(BaseDeployer): _fs (fsspec.filesystem): Filesystem for S3 protocol. """ - def __init__(self, endpoint, bucket_name): + def __init__(self, preview, endpoint, bucket_name): """Initialize S3Deployer with endpoint and bucket name. Args: endpoint (str): AWS endpoint of the hosted site. bucket_name (str): Name of the S3 bucket. """ super().__init__() + self.preview = preview self._endpoint = endpoint self._bucket_name = bucket_name self._path = f"{_S3_PROTOCOL}://{bucket_name}" diff --git a/package/kedro_viz/integrations/deployment/azure_deployer.py b/package/kedro_viz/integrations/deployment/azure_deployer.py index a147902545..254caea590 100644 --- a/package/kedro_viz/integrations/deployment/azure_deployer.py +++ b/package/kedro_viz/integrations/deployment/azure_deployer.py @@ -30,7 +30,7 @@ class AzureDeployer(BaseDeployer): _fs (fsspec.filesystem): Filesystem for Azure protocol. """ - def __init__(self, endpoint, bucket_name): + def __init__(self, preview, endpoint, bucket_name): """Initialize AzureBlobStorage with endpoint and bucket name. Args: @@ -38,6 +38,7 @@ def __init__(self, endpoint, bucket_name): bucket_name (str): Name of the AzureBlobStorage account. """ super().__init__() + self.preview = preview self._endpoint = endpoint self._bucket_name = bucket_name self._path = f"{_AZ_PROTOCOL}://$web" diff --git a/package/kedro_viz/integrations/deployment/base_deployer.py b/package/kedro_viz/integrations/deployment/base_deployer.py index 22c4fcc819..5f588f58d5 100644 --- a/package/kedro_viz/integrations/deployment/base_deployer.py +++ b/package/kedro_viz/integrations/deployment/base_deployer.py @@ -30,12 +30,13 @@ class BaseDeployer(abc.ABC): """ def __init__(self): + self.preview = False self._path = None self._fs = None def _upload_api_responses(self): """Write API responses to the build.""" - save_api_responses_to_fs(self._path, self._fs) + save_api_responses_to_fs(self._path, self._fs, self.preview) def _ingest_heap_analytics(self): """Ingest heap analytics to index file in the build.""" diff --git a/package/kedro_viz/integrations/deployment/deployer_factory.py b/package/kedro_viz/integrations/deployment/deployer_factory.py index 5215bf8c00..c60547ee9f 100644 --- a/package/kedro_viz/integrations/deployment/deployer_factory.py +++ b/package/kedro_viz/integrations/deployment/deployer_factory.py @@ -12,15 +12,15 @@ class DeployerFactory: """A class to handle creation of deployer class instances.""" @staticmethod - def create_deployer(platform, endpoint=None, bucket_name=None): + def create_deployer(platform, preview, endpoint=None, bucket_name=None): """Instantiate Kedro-viz deployer classes""" platform_name = platform.lower() if platform_name == "aws": - return AWSDeployer(endpoint, bucket_name) + return AWSDeployer(preview, endpoint, bucket_name) if platform_name == "azure": - return AzureDeployer(endpoint, bucket_name) + return AzureDeployer(preview, endpoint, bucket_name) if platform_name == "gcp": - return GCPDeployer(endpoint, bucket_name) + return GCPDeployer(preview, endpoint, bucket_name) if platform_name == "local": return LocalDeployer() raise ValueError( diff --git a/package/kedro_viz/models/flowchart.py b/package/kedro_viz/models/flowchart.py index e93418323c..33aa8377b1 100644 --- a/package/kedro_viz/models/flowchart.py +++ b/package/kedro_viz/models/flowchart.py @@ -769,6 +769,17 @@ class DataNodeMetadata(GraphNodeMetadata): description="The statistics for the data node.", ) + @model_validator(mode="before") + @classmethod + def check_is_preview_enabled_for_all_nodes_exists(cls, values): + assert "is_preview_enabled_for_all_nodes" in values + cls.set_is_preview_enabled_for_all_nodes(values["is_preview_enabled_for_all_nodes"]) + return values + + @classmethod + def set_is_preview_enabled_for_all_nodes(cls, is_preview_enabled_for_all_nodes): + cls.is_preview_enabled_for_all_nodes = is_preview_enabled_for_all_nodes + @model_validator(mode="before") @classmethod def check_data_node_exists(cls, values): @@ -806,6 +817,9 @@ def set_run_command(cls, _): @field_validator("preview") @classmethod def set_preview(cls, _): + if not cls.is_preview_enabled_for_all_nodes: + return None + if cls.data_node.is_preview_disabled() or not hasattr(cls.dataset, "preview"): return None @@ -829,6 +843,9 @@ def set_preview(cls, _): @field_validator("preview_type") @classmethod def set_preview_type(cls, _): + if not cls.is_preview_enabled_for_all_nodes: + return None + if cls.data_node.is_preview_disabled() or not hasattr(cls.dataset, "preview"): return None diff --git a/src/components/shareable-url-modal/shareable-url-modal.js b/src/components/shareable-url-modal/shareable-url-modal.js index 9d58bee238..58c582d4e9 100644 --- a/src/components/shareable-url-modal/shareable-url-modal.js +++ b/src/components/shareable-url-modal/shareable-url-modal.js @@ -104,7 +104,7 @@ const ShareableUrlModal = ({ onToggleModal, visible }) => { 'Content-Type': 'application/json', }, method: 'POST', - body: JSON.stringify(inputValues), + body: JSON.stringify({ ...inputValues, preview: false }), // preview appended to the body to disable the preview data }); const response = await request.json(); From dc51032b415e04d7f0d04d925b1882ddfd8c9656 Mon Sep 17 00:00:00 2001 From: Jitendra Gundaniya Date: Fri, 10 May 2024 19:51:08 +0100 Subject: [PATCH 08/35] Code review suggestions Signed-off-by: Jitendra Gundaniya --- package/kedro_viz/api/rest/requests.py | 2 +- package/kedro_viz/api/rest/responses.py | 16 +++++++----- package/kedro_viz/api/rest/router.py | 4 +-- .../integrations/deployment/aws_deployer.py | 3 +-- .../integrations/deployment/azure_deployer.py | 3 +-- .../integrations/deployment/base_deployer.py | 9 +++---- .../deployment/deployer_factory.py | 8 +++--- package/kedro_viz/models/flowchart.py | 25 ++++++++----------- 8 files changed, 33 insertions(+), 37 deletions(-) diff --git a/package/kedro_viz/api/rest/requests.py b/package/kedro_viz/api/rest/requests.py index 7a8c726e5e..373d966cc3 100644 --- a/package/kedro_viz/api/rest/requests.py +++ b/package/kedro_viz/api/rest/requests.py @@ -7,6 +7,6 @@ class DeployerConfiguration(BaseModel): """Credentials for Deployers.""" platform: str - preview: bool + is_preview_enabled: bool endpoint: str bucket_name: str diff --git a/package/kedro_viz/api/rest/responses.py b/package/kedro_viz/api/rest/responses.py index d06aeb2c0d..f2e7c97d96 100644 --- a/package/kedro_viz/api/rest/responses.py +++ b/package/kedro_viz/api/rest/responses.py @@ -320,7 +320,7 @@ def get_default_response() -> GraphAPIResponse: ) -def get_node_metadata_response(node_id: str, preview: bool = True): +def get_node_metadata_response(node_id: str): """API response for `/api/nodes/node_id`.""" node = data_access_manager.nodes.get_node_by_id(node_id) if not node: @@ -333,7 +333,7 @@ def get_node_metadata_response(node_id: str, preview: bool = True): return TaskNodeMetadata(task_node=node) if isinstance(node, DataNode): - return DataNodeMetadata(data_node=node, is_preview_enabled_for_all_nodes=preview) + return DataNodeMetadata(data_node=node) if isinstance(node, TranscodedDataNode): return TranscodedDataNodeMetadata(transcoded_data_node=node) @@ -419,12 +419,16 @@ def save_api_main_response_to_fs(main_path: str, remote_fs: Any): raise exc -def save_api_node_response_to_fs(nodes_path: str, remote_fs: Any, preview: bool): +def save_api_node_response_to_fs( + nodes_path: str, remote_fs: Any, is_preview_enabled: bool +): """Saves API /nodes/{node} response to a directory.""" + + DataNodeMetadata.set_preview_enabled(is_preview_enabled) for nodeId in data_access_manager.nodes.get_node_ids(): try: write_api_response_to_fs( - f"{nodes_path}/{nodeId}", get_node_metadata_response(nodeId, preview), remote_fs + f"{nodes_path}/{nodeId}", get_node_metadata_response(nodeId), remote_fs ) except Exception as exc: # pragma: no cover logger.exception( @@ -451,7 +455,7 @@ def save_api_pipeline_response_to_fs(pipelines_path: str, remote_fs: Any): raise exc -def save_api_responses_to_fs(path: str, remote_fs: Any, preview: bool): +def save_api_responses_to_fs(path: str, remote_fs: Any, is_preview_enabled: bool): """Saves all Kedro Viz API responses to a directory.""" try: logger.debug( @@ -469,7 +473,7 @@ def save_api_responses_to_fs(path: str, remote_fs: Any, preview: bool): remote_fs.makedirs(pipelines_path, exist_ok=True) save_api_main_response_to_fs(main_path, remote_fs) - save_api_node_response_to_fs(nodes_path, remote_fs, preview) + save_api_node_response_to_fs(nodes_path, remote_fs, is_preview_enabled) save_api_pipeline_response_to_fs(pipelines_path, remote_fs) except Exception as exc: # pragma: no cover diff --git a/package/kedro_viz/api/rest/router.py b/package/kedro_viz/api/rest/router.py index 7b381d02bc..4de98bf67d 100644 --- a/package/kedro_viz/api/rest/router.py +++ b/package/kedro_viz/api/rest/router.py @@ -61,9 +61,9 @@ async def get_single_pipeline_data(registered_pipeline_id: str): async def deploy_kedro_viz(input_values: DeployerConfiguration): try: deployer = DeployerFactory.create_deployer( - input_values.platform, input_values.preview,input_values.endpoint, input_values.bucket_name + input_values.platform, input_values.endpoint, input_values.bucket_name ) - deployer.deploy() + deployer.deploy(input_values.is_preview_enabled) response = { "message": "Website deployed on " f"{input_values.platform and input_values.platform.upper()}", diff --git a/package/kedro_viz/integrations/deployment/aws_deployer.py b/package/kedro_viz/integrations/deployment/aws_deployer.py index 93c27aeba5..2a9f4636f4 100644 --- a/package/kedro_viz/integrations/deployment/aws_deployer.py +++ b/package/kedro_viz/integrations/deployment/aws_deployer.py @@ -22,14 +22,13 @@ class AWSDeployer(BaseDeployer): _fs (fsspec.filesystem): Filesystem for S3 protocol. """ - def __init__(self, preview, endpoint, bucket_name): + def __init__(self, endpoint, bucket_name): """Initialize S3Deployer with endpoint and bucket name. Args: endpoint (str): AWS endpoint of the hosted site. bucket_name (str): Name of the S3 bucket. """ super().__init__() - self.preview = preview self._endpoint = endpoint self._bucket_name = bucket_name self._path = f"{_S3_PROTOCOL}://{bucket_name}" diff --git a/package/kedro_viz/integrations/deployment/azure_deployer.py b/package/kedro_viz/integrations/deployment/azure_deployer.py index 254caea590..a147902545 100644 --- a/package/kedro_viz/integrations/deployment/azure_deployer.py +++ b/package/kedro_viz/integrations/deployment/azure_deployer.py @@ -30,7 +30,7 @@ class AzureDeployer(BaseDeployer): _fs (fsspec.filesystem): Filesystem for Azure protocol. """ - def __init__(self, preview, endpoint, bucket_name): + def __init__(self, endpoint, bucket_name): """Initialize AzureBlobStorage with endpoint and bucket name. Args: @@ -38,7 +38,6 @@ def __init__(self, preview, endpoint, bucket_name): bucket_name (str): Name of the AzureBlobStorage account. """ super().__init__() - self.preview = preview self._endpoint = endpoint self._bucket_name = bucket_name self._path = f"{_AZ_PROTOCOL}://$web" diff --git a/package/kedro_viz/integrations/deployment/base_deployer.py b/package/kedro_viz/integrations/deployment/base_deployer.py index 5f588f58d5..5b39b83838 100644 --- a/package/kedro_viz/integrations/deployment/base_deployer.py +++ b/package/kedro_viz/integrations/deployment/base_deployer.py @@ -30,13 +30,12 @@ class BaseDeployer(abc.ABC): """ def __init__(self): - self.preview = False self._path = None self._fs = None - def _upload_api_responses(self): + def _upload_api_responses(self, is_preview_enabled: bool): """Write API responses to the build.""" - save_api_responses_to_fs(self._path, self._fs, self.preview) + save_api_responses_to_fs(self._path, self._fs, is_preview_enabled) def _ingest_heap_analytics(self): """Ingest heap analytics to index file in the build.""" @@ -99,9 +98,9 @@ def _upload_deploy_viz_metadata_file(self): logger.exception("Upload failed: %s ", exc) raise exc - def deploy(self): + def deploy(self, is_preview_enabled: bool = False): """Create and deploy all static files to local/remote file system""" - self._upload_api_responses() + self._upload_api_responses(is_preview_enabled) self._upload_static_files(_HTML_DIR) self._upload_deploy_viz_metadata_file() diff --git a/package/kedro_viz/integrations/deployment/deployer_factory.py b/package/kedro_viz/integrations/deployment/deployer_factory.py index c60547ee9f..5215bf8c00 100644 --- a/package/kedro_viz/integrations/deployment/deployer_factory.py +++ b/package/kedro_viz/integrations/deployment/deployer_factory.py @@ -12,15 +12,15 @@ class DeployerFactory: """A class to handle creation of deployer class instances.""" @staticmethod - def create_deployer(platform, preview, endpoint=None, bucket_name=None): + def create_deployer(platform, endpoint=None, bucket_name=None): """Instantiate Kedro-viz deployer classes""" platform_name = platform.lower() if platform_name == "aws": - return AWSDeployer(preview, endpoint, bucket_name) + return AWSDeployer(endpoint, bucket_name) if platform_name == "azure": - return AzureDeployer(preview, endpoint, bucket_name) + return AzureDeployer(endpoint, bucket_name) if platform_name == "gcp": - return GCPDeployer(preview, endpoint, bucket_name) + return GCPDeployer(endpoint, bucket_name) if platform_name == "local": return LocalDeployer() raise ValueError( diff --git a/package/kedro_viz/models/flowchart.py b/package/kedro_viz/models/flowchart.py index 33aa8377b1..608862a504 100644 --- a/package/kedro_viz/models/flowchart.py +++ b/package/kedro_viz/models/flowchart.py @@ -8,7 +8,7 @@ from enum import Enum from pathlib import Path from types import FunctionType -from typing import Any, Dict, List, Optional, Set, Union, cast +from typing import Any, ClassVar, Dict, List, Optional, Set, Union, cast from kedro.pipeline.node import Node as KedroNode @@ -735,6 +735,8 @@ class DataNodeMetadata(GraphNodeMetadata): data_node: DataNode = Field(..., exclude=True) + _is_preview_enabled: ClassVar[bool] = False + type: Optional[str] = Field( default=None, validate_default=True, description="The type of the data node" ) @@ -769,17 +771,6 @@ class DataNodeMetadata(GraphNodeMetadata): description="The statistics for the data node.", ) - @model_validator(mode="before") - @classmethod - def check_is_preview_enabled_for_all_nodes_exists(cls, values): - assert "is_preview_enabled_for_all_nodes" in values - cls.set_is_preview_enabled_for_all_nodes(values["is_preview_enabled_for_all_nodes"]) - return values - - @classmethod - def set_is_preview_enabled_for_all_nodes(cls, is_preview_enabled_for_all_nodes): - cls.is_preview_enabled_for_all_nodes = is_preview_enabled_for_all_nodes - @model_validator(mode="before") @classmethod def check_data_node_exists(cls, values): @@ -787,6 +778,10 @@ def check_data_node_exists(cls, values): cls.set_data_node_and_dataset(values["data_node"]) return values + @classmethod + def set_preview_enabled(cls, value: bool): + cls._is_preview_enabled = value + @classmethod def set_data_node_and_dataset(cls, data_node): cls.data_node = data_node @@ -817,7 +812,7 @@ def set_run_command(cls, _): @field_validator("preview") @classmethod def set_preview(cls, _): - if not cls.is_preview_enabled_for_all_nodes: + if not cls._is_preview_enabled: return None if cls.data_node.is_preview_disabled() or not hasattr(cls.dataset, "preview"): @@ -843,9 +838,9 @@ def set_preview(cls, _): @field_validator("preview_type") @classmethod def set_preview_type(cls, _): - if not cls.is_preview_enabled_for_all_nodes: + if not cls._is_preview_enabled: return None - + if cls.data_node.is_preview_disabled() or not hasattr(cls.dataset, "preview"): return None From f3b8007727749ea5c77d6e59f946d4969e5af94e Mon Sep 17 00:00:00 2001 From: Jitendra Gundaniya Date: Fri, 10 May 2024 20:01:09 +0100 Subject: [PATCH 09/35] temp code removed Signed-off-by: Jitendra Gundaniya --- src/components/shareable-url-modal/shareable-url-modal.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/shareable-url-modal/shareable-url-modal.js b/src/components/shareable-url-modal/shareable-url-modal.js index 58c582d4e9..9d58bee238 100644 --- a/src/components/shareable-url-modal/shareable-url-modal.js +++ b/src/components/shareable-url-modal/shareable-url-modal.js @@ -104,7 +104,7 @@ const ShareableUrlModal = ({ onToggleModal, visible }) => { 'Content-Type': 'application/json', }, method: 'POST', - body: JSON.stringify({ ...inputValues, preview: false }), // preview appended to the body to disable the preview data + body: JSON.stringify(inputValues), }); const response = await request.json(); From d8ef988ca76876b817d11a2be83c9dacdea3d98a Mon Sep 17 00:00:00 2001 From: ravi-kumar-pilla Date: Wed, 15 May 2024 16:27:33 -0500 Subject: [PATCH 10/35] refactor disable preview cli Signed-off-by: ravi-kumar-pilla --- package/kedro_viz/api/rest/responses.py | 17 +++++++++--- package/kedro_viz/data_access/managers.py | 12 --------- .../integrations/deployment/base_deployer.py | 10 ++++--- package/kedro_viz/launchers/cli.py | 10 +------ package/kedro_viz/models/flowchart.py | 26 +++++++++---------- package/kedro_viz/server.py | 9 ------- 6 files changed, 34 insertions(+), 50 deletions(-) diff --git a/package/kedro_viz/api/rest/responses.py b/package/kedro_viz/api/rest/responses.py index 13aaa05a6d..94ff986e08 100644 --- a/package/kedro_viz/api/rest/responses.py +++ b/package/kedro_viz/api/rest/responses.py @@ -419,8 +419,15 @@ def save_api_main_response_to_fs(main_path: str, remote_fs: Any): raise exc -def save_api_node_response_to_fs(nodes_path: str, remote_fs: Any): +def save_api_node_response_to_fs( + nodes_path: str, remote_fs: Any, is_preview_enabled_for_all_datasets: bool +): """Saves API /nodes/{node} response to a directory.""" + # Set if preview is enabled/disabled for all data nodes + DataNodeMetadata.set_is_preview_enabled_for_all_datasets( + is_preview_enabled_for_all_datasets + ) + for nodeId in data_access_manager.nodes.get_node_ids(): try: write_api_response_to_fs( @@ -451,7 +458,9 @@ def save_api_pipeline_response_to_fs(pipelines_path: str, remote_fs: Any): raise exc -def save_api_responses_to_fs(path: str, remote_fs: Any): +def save_api_responses_to_fs( + path: str, remote_fs: Any, is_preview_enabled_for_all_datasets: bool +): """Saves all Kedro Viz API responses to a directory.""" try: logger.debug( @@ -469,7 +478,9 @@ def save_api_responses_to_fs(path: str, remote_fs: Any): remote_fs.makedirs(pipelines_path, exist_ok=True) save_api_main_response_to_fs(main_path, remote_fs) - save_api_node_response_to_fs(nodes_path, remote_fs) + save_api_node_response_to_fs( + nodes_path, remote_fs, is_preview_enabled_for_all_datasets + ) save_api_pipeline_response_to_fs(pipelines_path, remote_fs) except Exception as exc: # pragma: no cover diff --git a/package/kedro_viz/data_access/managers.py b/package/kedro_viz/data_access/managers.py index 5ec03035cf..4e4e772e5a 100644 --- a/package/kedro_viz/data_access/managers.py +++ b/package/kedro_viz/data_access/managers.py @@ -64,7 +64,6 @@ def __init__(self): self.runs = RunsRepository() self.tracking_datasets = TrackingDatasetsRepository() self.dataset_stats = {} - self.is_preview_enabled_for_all_nodes = False def set_db_session(self, db_session_class: sessionmaker): """Set db session on repositories that need it.""" @@ -128,16 +127,6 @@ def add_dataset_stats(self, stats_dict: Dict): self.dataset_stats = stats_dict - def add_is_preview_enabled_for_all_nodes( - self, is_preview_enabled_for_all_nodes: bool - ): - """Add the preview flag to the data access manager. - Args: - is_preview_enabled_for_all_nodes: A boolean flag to indicate whether the preview is enabled for all the nodes. - """ - - self.is_preview_enabled_for_all_nodes = is_preview_enabled_for_all_nodes - def get_stats_for_data_node(self, data_node_name: str) -> Union[Dict, None]: """Returns the dataset statistics for the data node if found @@ -315,7 +304,6 @@ def add_dataset( dataset=obj, stats=self.get_stats_for_data_node(_strip_transcoding(dataset_name)), is_free_input=is_free_input, - is_preview_enabled_for_all_nodes=self.is_preview_enabled_for_all_nodes, ) graph_node = self.nodes.add_node(graph_node) graph_node.add_pipeline(registered_pipeline_id) diff --git a/package/kedro_viz/integrations/deployment/base_deployer.py b/package/kedro_viz/integrations/deployment/base_deployer.py index 22c4fcc819..ff4a3800ad 100644 --- a/package/kedro_viz/integrations/deployment/base_deployer.py +++ b/package/kedro_viz/integrations/deployment/base_deployer.py @@ -33,9 +33,11 @@ def __init__(self): self._path = None self._fs = None - def _upload_api_responses(self): + def _upload_api_responses(self, is_preview_enabled_for_all_datasets: bool): """Write API responses to the build.""" - save_api_responses_to_fs(self._path, self._fs) + save_api_responses_to_fs( + self._path, self._fs, is_preview_enabled_for_all_datasets + ) def _ingest_heap_analytics(self): """Ingest heap analytics to index file in the build.""" @@ -98,9 +100,9 @@ def _upload_deploy_viz_metadata_file(self): logger.exception("Upload failed: %s ", exc) raise exc - def deploy(self): + def deploy(self, is_preview_enabled_for_all_datasets: bool = False): """Create and deploy all static files to local/remote file system""" - self._upload_api_responses() + self._upload_api_responses(is_preview_enabled_for_all_datasets) self._upload_static_files(_HTML_DIR) self._upload_deploy_viz_metadata_file() diff --git a/package/kedro_viz/launchers/cli.py b/package/kedro_viz/launchers/cli.py index 47b64ab586..2e3b37e016 100644 --- a/package/kedro_viz/launchers/cli.py +++ b/package/kedro_viz/launchers/cli.py @@ -115,11 +115,6 @@ def viz(ctx): # pylint: disable=unused-argument help=PARAMS_ARG_HELP, callback=_split_params, ) -@click.option( - "--preview", - default=True, - help="A flag to enable/disable a quick preview of node datasets.", -) # pylint: disable=import-outside-toplevel, too-many-locals def run( host, @@ -132,7 +127,6 @@ def run( autoreload, include_hooks, params, - preview, ): """Launch local Kedro Viz instance""" from kedro_viz.server import run_server @@ -176,7 +170,6 @@ def run( "include_hooks": include_hooks, "package_name": PACKAGE_NAME, "extra_params": params, - "preview": preview, } if autoreload: run_process_kwargs = { @@ -384,14 +377,13 @@ def load_and_deploy_viz( try: load_and_populate_data( Path.cwd(), - is_preview_enabled_for_all_nodes=preview, include_hooks=include_hooks, package_name=package_name, ) # Start the deployment deployer = DeployerFactory.create_deployer(platform, endpoint, bucket_name) - deployer.deploy() + deployer.deploy(preview) except ( # pylint: disable=catching-non-exception diff --git a/package/kedro_viz/models/flowchart.py b/package/kedro_viz/models/flowchart.py index 8b228e4f72..5ad504297e 100644 --- a/package/kedro_viz/models/flowchart.py +++ b/package/kedro_viz/models/flowchart.py @@ -8,7 +8,7 @@ from enum import Enum from pathlib import Path from types import FunctionType -from typing import Any, Dict, List, Optional, Set, Union, cast +from typing import Any, ClassVar, Dict, List, Optional, Set, Union, cast from kedro.pipeline.node import Node as KedroNode @@ -227,7 +227,6 @@ def create_data_node( tags: Set[str], dataset: AbstractDataset, stats: Optional[Dict], - is_preview_enabled_for_all_nodes: bool, is_free_input: bool = False, ) -> Union["DataNode", "TranscodedDataNode"]: """Create a graph node of type data for a given Kedro Dataset instance. @@ -240,7 +239,6 @@ def create_data_node( dataset: A dataset in a Kedro pipeline. stats: The dictionary of dataset statistics, e.g. {"rows":2, "columns":3, "file_size":100} - is_preview_enabled_for_all_nodes: A flag to enable/disable a quick preview of node datasets. is_free_input: Whether the dataset is a free input in the pipeline Returns: An instance of DataNode. @@ -265,7 +263,6 @@ def create_data_node( kedro_obj=dataset, is_free_input=is_free_input, stats=stats, - is_preview_enabled_for_all_nodes=is_preview_enabled_for_all_nodes, ) @classmethod @@ -566,7 +563,6 @@ class DataNode(GraphNode): layer (Optional[str]): The layer that this data node belongs to. Defaults to `None`. is_free_input (bool): Determines whether the data node is a free input. Defaults to `False`. stats (Optional[Dict]): Statistics for the data node. Defaults to `None`. - is_preview_enabled_for_all_nodes (bool): A flag to enable/disable a quick preview of node datasets. Defaults to `False`. Raises: AssertionError: If kedro_obj, name are not supplied during instantiation @@ -580,10 +576,6 @@ class DataNode(GraphNode): ) stats: Optional[Dict] = Field(None, description="The statistics for the data node.") - is_preview_enabled_for_all_nodes: bool = Field( - False, description="A flag to enable/disable a quick preview of node datasets." - ) - dataset_type: Optional[str] = Field( default=None, validate_default=True, @@ -656,9 +648,7 @@ def get_preview_args(self): def is_preview_disabled(self): """Checks if the dataset has a preview disabled""" return ( - self.is_preview_enabled_for_all_nodes is False - or self.viz_metadata is not None - and self.viz_metadata.get("preview") is False + self.viz_metadata is not None and self.viz_metadata.get("preview") is False ) @@ -745,6 +735,8 @@ class DataNodeMetadata(GraphNodeMetadata): data_node: DataNode = Field(..., exclude=True) + is_preview_enabled_for_all_datasets: ClassVar[bool] = False + type: Optional[str] = Field( default=None, validate_default=True, description="The type of the data node" ) @@ -786,6 +778,10 @@ def check_data_node_exists(cls, values): cls.set_data_node_and_dataset(values["data_node"]) return values + @classmethod + def set_is_preview_enabled_for_all_datasets(cls, value: bool): + cls.is_preview_enabled_for_all_datasets = value + @classmethod def set_data_node_and_dataset(cls, data_node): cls.data_node = data_node @@ -816,7 +812,11 @@ def set_run_command(cls, _): @field_validator("preview") @classmethod def set_preview(cls, _): - if cls.data_node.is_preview_disabled() or not hasattr(cls.dataset, "preview"): + if ( + cls.data_node.is_preview_disabled() + or not hasattr(cls.dataset, "preview") + or not cls.is_preview_enabled_for_all_datasets + ): return None try: diff --git a/package/kedro_viz/server.py b/package/kedro_viz/server.py index f3168e6c8e..07b25d049c 100644 --- a/package/kedro_viz/server.py +++ b/package/kedro_viz/server.py @@ -30,7 +30,6 @@ def populate_data( pipelines: Dict[str, Pipeline], session_store: BaseSessionStore, stats_dict: Dict, - is_preview_enabled_for_all_nodes: bool, ): # pylint: disable=redefined-outer-name """Populate data repositories. Should be called once on application start if creating an api app from project. @@ -41,10 +40,6 @@ def populate_data( session_class = make_db_session_factory(session_store.location) data_access_manager.set_db_session(session_class) - data_access_manager.add_is_preview_enabled_for_all_nodes( - is_preview_enabled_for_all_nodes - ) - data_access_manager.add_catalog(catalog, pipelines) # add dataset stats before adding pipelines as the data nodes @@ -56,7 +51,6 @@ def populate_data( def load_and_populate_data( path: Path, - is_preview_enabled_for_all_nodes: bool, env: Optional[str] = None, include_hooks: bool = False, package_name: Optional[str] = None, @@ -87,14 +81,12 @@ def load_and_populate_data( pipelines, session_store, stats_dict, - is_preview_enabled_for_all_nodes, ) def run_server( host: str = DEFAULT_HOST, port: int = DEFAULT_PORT, - preview: bool = True, load_file: Optional[str] = None, save_file: Optional[str] = None, pipeline_name: Optional[str] = None, @@ -133,7 +125,6 @@ def run_server( if load_file is None: load_and_populate_data( path, - preview, env, include_hooks, package_name, From 3377832e04368e2c6914376acb25268881a845a7 Mon Sep 17 00:00:00 2001 From: ravi-kumar-pilla Date: Wed, 15 May 2024 17:04:31 -0500 Subject: [PATCH 11/35] fix lint Signed-off-by: ravi-kumar-pilla --- package/kedro_viz/api/rest/responses.py | 14 +++------- .../integrations/deployment/base_deployer.py | 10 +++---- package/kedro_viz/launchers/cli.py | 28 +++++++++++++------ package/kedro_viz/models/flowchart.py | 8 +++--- package/kedro_viz/server.py | 5 ++-- .../test_api/test_rest/test_responses.py | 18 +++++++----- .../test_integrations/test_base_deployer.py | 2 +- package/tests/test_launchers/test_cli.py | 12 +++++--- 8 files changed, 55 insertions(+), 42 deletions(-) diff --git a/package/kedro_viz/api/rest/responses.py b/package/kedro_viz/api/rest/responses.py index 94ff986e08..46d02173b1 100644 --- a/package/kedro_viz/api/rest/responses.py +++ b/package/kedro_viz/api/rest/responses.py @@ -420,13 +420,11 @@ def save_api_main_response_to_fs(main_path: str, remote_fs: Any): def save_api_node_response_to_fs( - nodes_path: str, remote_fs: Any, is_preview_enabled_for_all_datasets: bool + nodes_path: str, remote_fs: Any, is_datasets_previewed: bool ): """Saves API /nodes/{node} response to a directory.""" # Set if preview is enabled/disabled for all data nodes - DataNodeMetadata.set_is_preview_enabled_for_all_datasets( - is_preview_enabled_for_all_datasets - ) + DataNodeMetadata.set_is_datasets_previewed(is_datasets_previewed) for nodeId in data_access_manager.nodes.get_node_ids(): try: @@ -458,9 +456,7 @@ def save_api_pipeline_response_to_fs(pipelines_path: str, remote_fs: Any): raise exc -def save_api_responses_to_fs( - path: str, remote_fs: Any, is_preview_enabled_for_all_datasets: bool -): +def save_api_responses_to_fs(path: str, remote_fs: Any, is_datasets_previewed: bool): """Saves all Kedro Viz API responses to a directory.""" try: logger.debug( @@ -478,9 +474,7 @@ def save_api_responses_to_fs( remote_fs.makedirs(pipelines_path, exist_ok=True) save_api_main_response_to_fs(main_path, remote_fs) - save_api_node_response_to_fs( - nodes_path, remote_fs, is_preview_enabled_for_all_datasets - ) + save_api_node_response_to_fs(nodes_path, remote_fs, is_datasets_previewed) save_api_pipeline_response_to_fs(pipelines_path, remote_fs) except Exception as exc: # pragma: no cover diff --git a/package/kedro_viz/integrations/deployment/base_deployer.py b/package/kedro_viz/integrations/deployment/base_deployer.py index ff4a3800ad..555d25c4f2 100644 --- a/package/kedro_viz/integrations/deployment/base_deployer.py +++ b/package/kedro_viz/integrations/deployment/base_deployer.py @@ -33,11 +33,9 @@ def __init__(self): self._path = None self._fs = None - def _upload_api_responses(self, is_preview_enabled_for_all_datasets: bool): + def _upload_api_responses(self, is_datasets_previewed: bool): """Write API responses to the build.""" - save_api_responses_to_fs( - self._path, self._fs, is_preview_enabled_for_all_datasets - ) + save_api_responses_to_fs(self._path, self._fs, is_datasets_previewed) def _ingest_heap_analytics(self): """Ingest heap analytics to index file in the build.""" @@ -100,9 +98,9 @@ def _upload_deploy_viz_metadata_file(self): logger.exception("Upload failed: %s ", exc) raise exc - def deploy(self, is_preview_enabled_for_all_datasets: bool = False): + def deploy(self, is_datasets_previewed: bool = False): """Create and deploy all static files to local/remote file system""" - self._upload_api_responses(is_preview_enabled_for_all_datasets) + self._upload_api_responses(is_datasets_previewed) self._upload_static_files(_HTML_DIR) self._upload_deploy_viz_metadata_file() diff --git a/package/kedro_viz/launchers/cli.py b/package/kedro_viz/launchers/cli.py index 2e3b37e016..0a341cb23e 100644 --- a/package/kedro_viz/launchers/cli.py +++ b/package/kedro_viz/launchers/cli.py @@ -239,7 +239,7 @@ def run( default=False, help="A flag to enable/disable a quick preview of node datasets.", ) -def deploy(platform, endpoint, bucket_name, include_hooks, preview): +def deploy(platform, endpoint, bucket_name, include_hooks, is_datasets_previewed): """Deploy and host Kedro Viz on provided platform""" if not platform or platform.lower() not in SHAREABLEVIZ_SUPPORTED_PLATFORMS: display_cli_message( @@ -257,7 +257,13 @@ def deploy(platform, endpoint, bucket_name, include_hooks, preview): ) return - create_shareableviz_process(platform, preview, endpoint, bucket_name, include_hooks) + create_shareableviz_process( + platform, + is_datasets_previewed, + endpoint, + bucket_name, + include_hooks, + ) @viz.command(context_settings={"help_option_names": ["-h", "--help"]}) @@ -271,14 +277,20 @@ def deploy(platform, endpoint, bucket_name, include_hooks, preview): default=False, help="A flag to enable/disable a quick preview of node datasets.", ) -def build(include_hooks, preview): +def build(include_hooks, is_datasets_previewed): """Create build directory of local Kedro Viz instance with Kedro project data""" - create_shareableviz_process("local", preview, include_hooks=include_hooks) + create_shareableviz_process( + "local", is_datasets_previewed, include_hooks=include_hooks + ) def create_shareableviz_process( - platform, preview, endpoint=None, bucket_name=None, include_hooks=False + platform, + is_datasets_previewed, + endpoint=None, + bucket_name=None, + include_hooks=False, ): """Creates platform specific deployer process""" try: @@ -289,7 +301,7 @@ def create_shareableviz_process( target=load_and_deploy_viz, args=( platform, - preview, + is_datasets_previewed, endpoint, bucket_name, include_hooks, @@ -365,7 +377,7 @@ def create_shareableviz_process( def load_and_deploy_viz( platform, - preview, + is_datasets_previewed, endpoint, bucket_name, include_hooks, @@ -383,7 +395,7 @@ def load_and_deploy_viz( # Start the deployment deployer = DeployerFactory.create_deployer(platform, endpoint, bucket_name) - deployer.deploy(preview) + deployer.deploy(is_datasets_previewed) except ( # pylint: disable=catching-non-exception diff --git a/package/kedro_viz/models/flowchart.py b/package/kedro_viz/models/flowchart.py index 5ad504297e..75a759a178 100644 --- a/package/kedro_viz/models/flowchart.py +++ b/package/kedro_viz/models/flowchart.py @@ -735,7 +735,7 @@ class DataNodeMetadata(GraphNodeMetadata): data_node: DataNode = Field(..., exclude=True) - is_preview_enabled_for_all_datasets: ClassVar[bool] = False + is_datasets_previewed: ClassVar[bool] = False type: Optional[str] = Field( default=None, validate_default=True, description="The type of the data node" @@ -779,8 +779,8 @@ def check_data_node_exists(cls, values): return values @classmethod - def set_is_preview_enabled_for_all_datasets(cls, value: bool): - cls.is_preview_enabled_for_all_datasets = value + def set_is_datasets_previewed(cls, value: bool): + cls.is_datasets_previewed = value @classmethod def set_data_node_and_dataset(cls, data_node): @@ -815,7 +815,7 @@ def set_preview(cls, _): if ( cls.data_node.is_preview_disabled() or not hasattr(cls.dataset, "preview") - or not cls.is_preview_enabled_for_all_datasets + or not cls.is_datasets_previewed ): return None diff --git a/package/kedro_viz/server.py b/package/kedro_viz/server.py index 07b25d049c..445da9bbe0 100644 --- a/package/kedro_viz/server.py +++ b/package/kedro_viz/server.py @@ -131,9 +131,10 @@ def run_server( pipeline_name, extra_params, ) - + # [TODO: As we can do this with `kedro viz build`, + # we need to shift this feature outside of kedro viz run] if save_file: - save_api_responses_to_fs(save_file, fsspec.filesystem("file")) + save_api_responses_to_fs(save_file, fsspec.filesystem("file"), True) app = apps.create_api_app_from_project(path, autoreload) else: diff --git a/package/tests/test_api/test_rest/test_responses.py b/package/tests/test_api/test_rest/test_responses.py index 5e5d32a4c2..4c1194870a 100644 --- a/package/tests/test_api/test_rest/test_responses.py +++ b/package/tests/test_api/test_rest/test_responses.py @@ -948,7 +948,7 @@ def test_save_api_node_response_to_fs(self, mocker): ) remote_fs = Mock() - save_api_node_response_to_fs(nodes_path, remote_fs) + save_api_node_response_to_fs(nodes_path, remote_fs, False) assert mock_write_api_response_to_fs.call_count == len(nodeIds) assert mock_get_node_metadata_response.call_count == len(nodeIds) @@ -1000,14 +1000,16 @@ def test_save_api_pipeline_response_to_fs(self, mocker): mock_write_api_response_to_fs.assert_has_calls(expected_calls, any_order=True) @pytest.mark.parametrize( - "file_path, protocol", + "file_path, protocol, is_datasets_previewed", [ - ("s3://shareableviz", "s3"), - ("abfs://shareableviz", "abfs"), - ("shareableviz", "file"), + ("s3://shareableviz", "s3", True), + ("abfs://shareableviz", "abfs", False), + ("shareableviz", "file", True), ], ) - def test_save_api_responses_to_fs(self, file_path, protocol, mocker): + def test_save_api_responses_to_fs( + self, file_path, protocol, is_datasets_previewed, mocker + ): mock_api_main_response_to_fs = mocker.patch( "kedro_viz.api.rest.responses.save_api_main_response_to_fs" ) @@ -1021,7 +1023,9 @@ def test_save_api_responses_to_fs(self, file_path, protocol, mocker): mock_filesystem = mocker.patch("fsspec.filesystem") mock_filesystem.return_value.protocol = protocol - save_api_responses_to_fs(file_path, mock_filesystem.return_value) + save_api_responses_to_fs( + file_path, mock_filesystem.return_value, is_datasets_previewed + ) mock_api_main_response_to_fs.assert_called_once_with( f"{file_path}/api/main", mock_filesystem.return_value diff --git a/package/tests/test_integrations/test_base_deployer.py b/package/tests/test_integrations/test_base_deployer.py index d05947408e..e1ebfd671b 100644 --- a/package/tests/test_integrations/test_base_deployer.py +++ b/package/tests/test_integrations/test_base_deployer.py @@ -14,7 +14,7 @@ def test_upload_api_responses(self, mocker): "kedro_viz.integrations.deployment.base_deployer.save_api_responses_to_fs" ) build = ConcreteBaseDeployer() - build._upload_api_responses() + build._upload_api_responses(False) save_api_responses_to_fs_mock.assert_called_once_with(build._path, build._fs) diff --git a/package/tests/test_launchers/test_cli.py b/package/tests/test_launchers/test_cli.py index 4ad1b6dd86..218b7e3258 100755 --- a/package/tests/test_launchers/test_cli.py +++ b/package/tests/test_launchers/test_cli.py @@ -670,10 +670,11 @@ def test_create_shareableviz_process( @pytest.mark.parametrize( - "platform, endpoint, bucket_name, include_hooks, package_name", + "platform, is_datasets_previewed, endpoint, bucket_name, include_hooks, package_name", [ ( "azure", + False, "https://example-bucket.web.core.windows.net", "example-bucket", False, @@ -681,17 +682,19 @@ def test_create_shareableviz_process( ), ( "aws", + True, "http://example-bucket.s3-website.us-east-2.amazonaws.com/", "example-bucket", True, "demo_project", ), - ("gcp", "http://34.120.87.227/", "example-bucket", False, "demo_project"), - ("local", None, None, True, "demo_project"), + ("gcp", True, "http://34.120.87.227/", "example-bucket", False, "demo_project"), + ("local", False, None, None, True, "demo_project"), ], ) def test_load_and_deploy_viz_success( platform, + is_datasets_previewed, endpoint, bucket_name, include_hooks, @@ -707,6 +710,7 @@ def test_load_and_deploy_viz_success( cli.load_and_deploy_viz( platform, + is_datasets_previewed, endpoint, bucket_name, include_hooks, @@ -721,5 +725,5 @@ def test_load_and_deploy_viz_success( mock_DeployerFactory.create_deployer.assert_called_once_with( platform, endpoint, bucket_name ) - deployer_mock.deploy.assert_called_once() + deployer_mock.deploy.assert_called_once_with(is_datasets_previewed) mock_click_echo.echo.assert_not_called() From 7a317293cce90b3115c755900351e98ec8182673 Mon Sep 17 00:00:00 2001 From: ravi-kumar-pilla Date: Wed, 15 May 2024 17:09:26 -0500 Subject: [PATCH 12/35] fix lint Signed-off-by: ravi-kumar-pilla --- package/kedro_viz/server.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/package/kedro_viz/server.py b/package/kedro_viz/server.py index 445da9bbe0..384a3545dc 100644 --- a/package/kedro_viz/server.py +++ b/package/kedro_viz/server.py @@ -75,13 +75,7 @@ def load_and_populate_data( ) # Creates data repositories which are used by Kedro Viz Backend APIs - populate_data( - data_access_manager, - catalog, - pipelines, - session_store, - stats_dict, - ) + populate_data(data_access_manager, catalog, pipelines, session_store, stats_dict) def run_server( From 78f44bb4a9323bf374c338ba36f98dc5bbee481c Mon Sep 17 00:00:00 2001 From: ravi-kumar-pilla Date: Thu, 16 May 2024 15:01:38 -0500 Subject: [PATCH 13/35] testing 1 Signed-off-by: ravi-kumar-pilla --- package/kedro_viz/launchers/cli.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/package/kedro_viz/launchers/cli.py b/package/kedro_viz/launchers/cli.py index 0a341cb23e..09c27f21cd 100644 --- a/package/kedro_viz/launchers/cli.py +++ b/package/kedro_viz/launchers/cli.py @@ -237,9 +237,9 @@ def run( @click.option( "--preview", default=False, - help="A flag to enable/disable a quick preview of node datasets.", + help="Enable/disable a quick preview of node datasets.", ) -def deploy(platform, endpoint, bucket_name, include_hooks, is_datasets_previewed): +def deploy(platform, endpoint, bucket_name, include_hooks, preview): """Deploy and host Kedro Viz on provided platform""" if not platform or platform.lower() not in SHAREABLEVIZ_SUPPORTED_PLATFORMS: display_cli_message( @@ -259,7 +259,7 @@ def deploy(platform, endpoint, bucket_name, include_hooks, is_datasets_previewed create_shareableviz_process( platform, - is_datasets_previewed, + preview, endpoint, bucket_name, include_hooks, @@ -275,14 +275,12 @@ def deploy(platform, endpoint, bucket_name, include_hooks, is_datasets_previewed @click.option( "--preview", default=False, - help="A flag to enable/disable a quick preview of node datasets.", + help="Enable/disable a quick preview of node datasets.", ) -def build(include_hooks, is_datasets_previewed): +def build(include_hooks, preview): """Create build directory of local Kedro Viz instance with Kedro project data""" - create_shareableviz_process( - "local", is_datasets_previewed, include_hooks=include_hooks - ) + create_shareableviz_process("local", preview, include_hooks=include_hooks) def create_shareableviz_process( From 663924c81252681f6dd09bb52126f11f3e180d42 Mon Sep 17 00:00:00 2001 From: ravi-kumar-pilla Date: Thu, 16 May 2024 15:23:01 -0500 Subject: [PATCH 14/35] change help text Signed-off-by: ravi-kumar-pilla --- package/kedro_viz/launchers/cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package/kedro_viz/launchers/cli.py b/package/kedro_viz/launchers/cli.py index 09c27f21cd..449242e7cd 100644 --- a/package/kedro_viz/launchers/cli.py +++ b/package/kedro_viz/launchers/cli.py @@ -237,7 +237,7 @@ def run( @click.option( "--preview", default=False, - help="Enable/disable a quick preview of node datasets.", + help="Enable/disable preview of node datasets.", ) def deploy(platform, endpoint, bucket_name, include_hooks, preview): """Deploy and host Kedro Viz on provided platform""" @@ -275,7 +275,7 @@ def deploy(platform, endpoint, bucket_name, include_hooks, preview): @click.option( "--preview", default=False, - help="Enable/disable a quick preview of node datasets.", + help="Enable/disable preview of node datasets.", ) def build(include_hooks, preview): """Create build directory of local Kedro Viz instance with Kedro project data""" From defbf6640992fb39a347fdc9aa931ef16ec13a2e Mon Sep 17 00:00:00 2001 From: ravi-kumar-pilla Date: Thu, 16 May 2024 15:25:33 -0500 Subject: [PATCH 15/35] change help text Signed-off-by: ravi-kumar-pilla --- package/kedro_viz/launchers/cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package/kedro_viz/launchers/cli.py b/package/kedro_viz/launchers/cli.py index 449242e7cd..af8909d603 100644 --- a/package/kedro_viz/launchers/cli.py +++ b/package/kedro_viz/launchers/cli.py @@ -237,7 +237,7 @@ def run( @click.option( "--preview", default=False, - help="Enable/disable preview of node datasets.", + help="Enable/disable the preview of all datasets.", ) def deploy(platform, endpoint, bucket_name, include_hooks, preview): """Deploy and host Kedro Viz on provided platform""" @@ -275,7 +275,7 @@ def deploy(platform, endpoint, bucket_name, include_hooks, preview): @click.option( "--preview", default=False, - help="Enable/disable preview of node datasets.", + help="Enable/disable the preview of all datasets.", ) def build(include_hooks, preview): """Create build directory of local Kedro Viz instance with Kedro project data""" From 91951508f07369e96e3d1c7366b7f97b130e2efe Mon Sep 17 00:00:00 2001 From: ravi-kumar-pilla Date: Thu, 16 May 2024 17:27:49 -0500 Subject: [PATCH 16/35] update tests Signed-off-by: ravi-kumar-pilla --- .../test_api/test_rest/test_responses.py | 4 +- .../test_integrations/test_base_deployer.py | 4 +- package/tests/test_launchers/test_cli.py | 45 +++++++++++++++++-- package/tests/test_models/test_flowchart.py | 6 ++- package/tests/test_server.py | 2 +- 5 files changed, 52 insertions(+), 9 deletions(-) mode change 100644 => 100755 package/tests/test_integrations/test_base_deployer.py diff --git a/package/tests/test_api/test_rest/test_responses.py b/package/tests/test_api/test_rest/test_responses.py index 01881ab626..7464efefd9 100644 --- a/package/tests/test_api/test_rest/test_responses.py +++ b/package/tests/test_api/test_rest/test_responses.py @@ -1062,7 +1062,9 @@ def test_save_api_responses_to_fs( f"{file_path}/api/main", mock_filesystem.return_value ) mock_api_node_response_to_fs.assert_called_once_with( - f"{file_path}/api/nodes", mock_filesystem.return_value + f"{file_path}/api/nodes", + mock_filesystem.return_value, + is_datasets_previewed, ) mock_api_pipeline_response_to_fs.assert_called_once_with( f"{file_path}/api/pipelines", mock_filesystem.return_value diff --git a/package/tests/test_integrations/test_base_deployer.py b/package/tests/test_integrations/test_base_deployer.py old mode 100644 new mode 100755 index e1ebfd671b..a933e6229a --- a/package/tests/test_integrations/test_base_deployer.py +++ b/package/tests/test_integrations/test_base_deployer.py @@ -16,7 +16,9 @@ def test_upload_api_responses(self, mocker): build = ConcreteBaseDeployer() build._upload_api_responses(False) - save_api_responses_to_fs_mock.assert_called_once_with(build._path, build._fs) + save_api_responses_to_fs_mock.assert_called_once_with( + build._path, build._fs, False + ) def test_upload_static_files(self, mocker): mocker.patch("fsspec.filesystem") diff --git a/package/tests/test_launchers/test_cli.py b/package/tests/test_launchers/test_cli.py index 218b7e3258..7ff37d9a5a 100755 --- a/package/tests/test_launchers/test_cli.py +++ b/package/tests/test_launchers/test_cli.py @@ -445,6 +445,26 @@ def test_viz_command_group(mocker, mock_click_echo): "include_hooks": True, }, ), + ( + [ + "viz", + "deploy", + "--platform", + "aws", + "--endpoint", + "http://example-bucket.s3-website.us-east-2.amazonaws.com/", + "--bucket-name", + "example-bucket", + "--preview", + "true", + ], + { + "platform": "aws", + "endpoint": "http://example-bucket.s3-website.us-east-2.amazonaws.com/", + "bucket_name": "example-bucket", + "preview": True, + }, + ), ], ) def test_viz_deploy_valid_endpoint_and_bucket(command_options, deployer_args, mocker): @@ -461,6 +481,7 @@ def test_viz_deploy_valid_endpoint_and_bucket(command_options, deployer_args, mo create_shareableviz_process_mock.assert_called_once_with( deployer_args.get("platform"), + deployer_args.get("preview", False), deployer_args.get("endpoint"), deployer_args.get("bucket_name"), deployer_args.get("include_hooks", False), @@ -539,6 +560,10 @@ def test_viz_deploy_invalid_endpoint(mocker, mock_click_echo): ["viz", "build", "--include-hooks"], {"platform": "local", "include_hooks": True}, ), + ( + ["viz", "build", "--preview", "true"], + {"platform": "local", "preview": True}, + ), ], ) def test_successful_build_with_existing_static_files( @@ -557,15 +582,18 @@ def test_successful_build_with_existing_static_files( create_shareableviz_process_mock.assert_called_once_with( build_args.get("platform"), + build_args.get("preview", False), include_hooks=build_args.get("include_hooks", False), ) @pytest.mark.parametrize( - "platform, endpoint, bucket_name, include_hooks, process_completed_value", + "platform, is_datasets_previewed, endpoint, bucket_name," + "include_hooks, process_completed_value", [ ( "azure", + True, "https://example-bucket.web.core.windows.net", "example-bucket", True, @@ -573,6 +601,7 @@ def test_successful_build_with_existing_static_files( ), ( "aws", + True, "http://example-bucket.s3-website.us-east-2.amazonaws.com/", "example-bucket", True, @@ -580,14 +609,16 @@ def test_successful_build_with_existing_static_files( ), ( "gcp", + False, "http://34.120.87.227/", "example-bucket", False, 1, ), - ("local", None, None, False, 1), + ("local", False, None, None, False, 1), ( "azure", + True, "https://example-bucket.web.core.windows.net", "example-bucket", False, @@ -595,6 +626,7 @@ def test_successful_build_with_existing_static_files( ), ( "aws", + False, "http://example-bucket.s3-website.us-east-2.amazonaws.com/", "example-bucket", False, @@ -602,16 +634,18 @@ def test_successful_build_with_existing_static_files( ), ( "gcp", + True, "http://34.120.87.227/", "example-bucket", True, 0, ), - ("local", None, None, True, 0), + ("local", True, None, None, True, 0), ], ) def test_create_shareableviz_process( platform, + is_datasets_previewed, endpoint, bucket_name, include_hooks, @@ -624,13 +658,16 @@ def test_create_shareableviz_process( mock_click_echo, ): mock_process_completed.return_value.value = process_completed_value - cli.create_shareableviz_process(platform, endpoint, bucket_name, include_hooks) + cli.create_shareableviz_process( + platform, is_datasets_previewed, endpoint, bucket_name, include_hooks + ) # Assert the mocks were called as expected mock_viz_deploy_process.assert_called_once_with( target=mock_viz_load_and_deploy, args=( platform, + is_datasets_previewed, endpoint, bucket_name, include_hooks, diff --git a/package/tests/test_models/test_flowchart.py b/package/tests/test_models/test_flowchart.py index d9f7a75235..a81e4c45c1 100644 --- a/package/tests/test_models/test_flowchart.py +++ b/package/tests/test_models/test_flowchart.py @@ -404,6 +404,8 @@ def test_preview_data_node_metadata(self, example_data_node): [12345, "80%", "Barbados"], ], } + + DataNodeMetadata.set_is_datasets_previewed(True) preview_node_metadata = DataNodeMetadata(data_node=example_data_node) assert preview_node_metadata.preview == expected_preview_data @@ -444,10 +446,10 @@ def test_preview_default_data_node_metadata( assert preview_node_metadata.preview == expected_preview_data def test_preview_data_node_metadata_not_exist(self, example_data_node, mocker): - mocker.patch("kedro_datasets.pandas.CSVDataset.preview", return_value=False) + mocker.patch("kedro_datasets.pandas.CSVDataset.preview", return_value=None) preview_node_metadata = DataNodeMetadata(data_node=example_data_node) - assert preview_node_metadata.preview is False + assert preview_node_metadata.preview is None def test_transcoded_data_node_metadata(self): dataset = CSVDataset(filepath="/tmp/dataset.csv") diff --git a/package/tests/test_server.py b/package/tests/test_server.py index 391f9511c7..33fe6f2e1b 100644 --- a/package/tests/test_server.py +++ b/package/tests/test_server.py @@ -156,5 +156,5 @@ def test_save_file(self, tmp_path, mocker): save_file = tmp_path / "save.json" run_server(save_file=save_file) save_api_responses_to_fs_mock.assert_called_once_with( - save_file, mock_filesystem.return_value + save_file, mock_filesystem.return_value, True ) From c4c7375d1003530ae42913697ef64d8a42eb3d43 Mon Sep 17 00:00:00 2001 From: ravi-kumar-pilla Date: Thu, 16 May 2024 17:33:31 -0500 Subject: [PATCH 17/35] format help text Signed-off-by: ravi-kumar-pilla --- package/kedro_viz/launchers/cli.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/package/kedro_viz/launchers/cli.py b/package/kedro_viz/launchers/cli.py index af8909d603..37491cfce7 100644 --- a/package/kedro_viz/launchers/cli.py +++ b/package/kedro_viz/launchers/cli.py @@ -237,7 +237,7 @@ def run( @click.option( "--preview", default=False, - help="Enable/disable the preview of all datasets.", + help="Enable/disable preview for all the datasets.", ) def deploy(platform, endpoint, bucket_name, include_hooks, preview): """Deploy and host Kedro Viz on provided platform""" @@ -275,7 +275,7 @@ def deploy(platform, endpoint, bucket_name, include_hooks, preview): @click.option( "--preview", default=False, - help="Enable/disable the preview of all datasets.", + help="Enable/disable preview for all the datasets.", ) def build(include_hooks, preview): """Create build directory of local Kedro Viz instance with Kedro project data""" @@ -386,9 +386,7 @@ def load_and_deploy_viz( """Loads Kedro Project data, creates a deployer and deploys to a platform""" try: load_and_populate_data( - Path.cwd(), - include_hooks=include_hooks, - package_name=package_name, + Path.cwd(), include_hooks=include_hooks, package_name=package_name ) # Start the deployment From 6eaaaebeca7fb4af9bb1b9263b7ef11fdc2abee1 Mon Sep 17 00:00:00 2001 From: ravi-kumar-pilla Date: Thu, 16 May 2024 17:40:37 -0500 Subject: [PATCH 18/35] change default preview for run Signed-off-by: ravi-kumar-pilla --- package/kedro_viz/models/flowchart.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package/kedro_viz/models/flowchart.py b/package/kedro_viz/models/flowchart.py index 75a759a178..ff891bc779 100644 --- a/package/kedro_viz/models/flowchart.py +++ b/package/kedro_viz/models/flowchart.py @@ -735,7 +735,7 @@ class DataNodeMetadata(GraphNodeMetadata): data_node: DataNode = Field(..., exclude=True) - is_datasets_previewed: ClassVar[bool] = False + is_datasets_previewed: ClassVar[bool] = True type: Optional[str] = Field( default=None, validate_default=True, description="The type of the data node" From 8372ec3b641b65c2693972e171721d714acd742d Mon Sep 17 00:00:00 2001 From: ravi-kumar-pilla Date: Thu, 16 May 2024 17:59:40 -0500 Subject: [PATCH 19/35] update release note Signed-off-by: ravi-kumar-pilla --- RELEASE.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/RELEASE.md b/RELEASE.md index c3cd6e3edf..a868f9b2ed 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -9,6 +9,8 @@ Please follow the established format: ## Major features and improvements +- Enable/disable preview for all the datasets when publishing Kedro-Viz via CLI. (#1894) + ## Bug fixes and other changes - Upgrade the gitpod workspace-full to a newer version which includes both Node 18 and Python 3.11.5. (#1862) From 2f67e78ba3b8166b5838db4a92ee461b2ac75a76 Mon Sep 17 00:00:00 2001 From: ravi-kumar-pilla Date: Thu, 16 May 2024 20:03:53 -0500 Subject: [PATCH 20/35] address vale suggestion Signed-off-by: ravi-kumar-pilla --- RELEASE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE.md b/RELEASE.md index a868f9b2ed..2ede3cee15 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -9,7 +9,7 @@ Please follow the established format: ## Major features and improvements -- Enable/disable preview for all the datasets when publishing Kedro-Viz via CLI. (#1894) +- Enable/disable preview for all the datasets when publishing Kedro-Viz with CLI. (#1894) ## Bug fixes and other changes From e465101e92648fdaedbb373ea04a54238fdd5f5a Mon Sep 17 00:00:00 2001 From: ravi-kumar-pilla Date: Thu, 16 May 2024 20:14:43 -0500 Subject: [PATCH 21/35] fix lint Signed-off-by: ravi-kumar-pilla --- package/kedro_viz/api/rest/requests.py | 2 +- package/kedro_viz/api/rest/responses.py | 2 +- package/kedro_viz/api/rest/router.py | 2 +- package/kedro_viz/models/flowchart.py | 9 +++++---- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/package/kedro_viz/api/rest/requests.py b/package/kedro_viz/api/rest/requests.py index 373d966cc3..a6ce949d3d 100644 --- a/package/kedro_viz/api/rest/requests.py +++ b/package/kedro_viz/api/rest/requests.py @@ -7,6 +7,6 @@ class DeployerConfiguration(BaseModel): """Credentials for Deployers.""" platform: str - is_preview_enabled: bool + is_datasets_previewed: bool endpoint: str bucket_name: str diff --git a/package/kedro_viz/api/rest/responses.py b/package/kedro_viz/api/rest/responses.py index 1d70f99844..46165a79c8 100644 --- a/package/kedro_viz/api/rest/responses.py +++ b/package/kedro_viz/api/rest/responses.py @@ -482,4 +482,4 @@ def save_api_responses_to_fs(path: str, remote_fs: Any, is_datasets_previewed: b logger.exception( "An error occurred while preparing data for saving. Error: %s", str(exc) ) - raise exc \ No newline at end of file + raise exc diff --git a/package/kedro_viz/api/rest/router.py b/package/kedro_viz/api/rest/router.py index 4de98bf67d..c72003eb60 100644 --- a/package/kedro_viz/api/rest/router.py +++ b/package/kedro_viz/api/rest/router.py @@ -63,7 +63,7 @@ async def deploy_kedro_viz(input_values: DeployerConfiguration): deployer = DeployerFactory.create_deployer( input_values.platform, input_values.endpoint, input_values.bucket_name ) - deployer.deploy(input_values.is_preview_enabled) + deployer.deploy(input_values.is_datasets_previewed) response = { "message": "Website deployed on " f"{input_values.platform and input_values.platform.upper()}", diff --git a/package/kedro_viz/models/flowchart.py b/package/kedro_viz/models/flowchart.py index 4c7984d1e2..45fd1dfdad 100644 --- a/package/kedro_viz/models/flowchart.py +++ b/package/kedro_viz/models/flowchart.py @@ -839,10 +839,11 @@ def set_preview(cls, _): @field_validator("preview_type") @classmethod def set_preview_type(cls, _): - if not cls._is_preview_enabled: - return None - - if cls.data_node.is_preview_disabled() or not hasattr(cls.dataset, "preview"): + if ( + cls.data_node.is_preview_disabled() + or not hasattr(cls.dataset, "preview") + or not cls.is_datasets_previewed + ): return None try: From 12eabd19559263b22a82bda059705b1ac0852ebf Mon Sep 17 00:00:00 2001 From: ravi-kumar-pilla Date: Thu, 16 May 2024 23:01:42 -0500 Subject: [PATCH 22/35] fix pytests and linter Signed-off-by: ravi-kumar-pilla --- package/kedro_viz/api/rest/responses.py | 8 ++++---- .../integrations/deployment/base_deployer.py | 8 ++++---- package/kedro_viz/launchers/cli.py | 8 ++++---- package/kedro_viz/models/flowchart.py | 14 +++++++++----- package/tests/conftest.py | 7 ++++++- .../tests/test_api/test_rest/test_responses.py | 8 ++++---- .../test_integrations/test_azure_deployer.py | 2 +- .../test_integrations/test_deployer_factory.py | 3 ++- .../tests/test_integrations/test_gcp_deployer.py | 2 +- package/tests/test_launchers/test_cli.py | 16 ++++++++-------- package/tests/test_models/test_flowchart.py | 9 ++++++++- 11 files changed, 51 insertions(+), 34 deletions(-) diff --git a/package/kedro_viz/api/rest/responses.py b/package/kedro_viz/api/rest/responses.py index 46165a79c8..2c2ce33424 100644 --- a/package/kedro_viz/api/rest/responses.py +++ b/package/kedro_viz/api/rest/responses.py @@ -421,11 +421,11 @@ def save_api_main_response_to_fs(main_path: str, remote_fs: Any): def save_api_node_response_to_fs( - nodes_path: str, remote_fs: Any, is_datasets_previewed: bool + nodes_path: str, remote_fs: Any, are_datasets_previewable: bool ): """Saves API /nodes/{node} response to a directory.""" # Set if preview is enabled/disabled for all data nodes - DataNodeMetadata.set_is_datasets_previewed(is_datasets_previewed) + DataNodeMetadata.set_are_datasets_previewable(are_datasets_previewable) for nodeId in data_access_manager.nodes.get_node_ids(): try: @@ -457,7 +457,7 @@ def save_api_pipeline_response_to_fs(pipelines_path: str, remote_fs: Any): raise exc -def save_api_responses_to_fs(path: str, remote_fs: Any, is_datasets_previewed: bool): +def save_api_responses_to_fs(path: str, remote_fs: Any, are_datasets_previewable: bool): """Saves all Kedro Viz API responses to a directory.""" try: logger.debug( @@ -475,7 +475,7 @@ def save_api_responses_to_fs(path: str, remote_fs: Any, is_datasets_previewed: b remote_fs.makedirs(pipelines_path, exist_ok=True) save_api_main_response_to_fs(main_path, remote_fs) - save_api_node_response_to_fs(nodes_path, remote_fs, is_datasets_previewed) + save_api_node_response_to_fs(nodes_path, remote_fs, are_datasets_previewable) save_api_pipeline_response_to_fs(pipelines_path, remote_fs) except Exception as exc: # pragma: no cover diff --git a/package/kedro_viz/integrations/deployment/base_deployer.py b/package/kedro_viz/integrations/deployment/base_deployer.py index 555d25c4f2..73e23d815a 100644 --- a/package/kedro_viz/integrations/deployment/base_deployer.py +++ b/package/kedro_viz/integrations/deployment/base_deployer.py @@ -33,9 +33,9 @@ def __init__(self): self._path = None self._fs = None - def _upload_api_responses(self, is_datasets_previewed: bool): + def _upload_api_responses(self, are_datasets_previewable: bool): """Write API responses to the build.""" - save_api_responses_to_fs(self._path, self._fs, is_datasets_previewed) + save_api_responses_to_fs(self._path, self._fs, are_datasets_previewable) def _ingest_heap_analytics(self): """Ingest heap analytics to index file in the build.""" @@ -98,9 +98,9 @@ def _upload_deploy_viz_metadata_file(self): logger.exception("Upload failed: %s ", exc) raise exc - def deploy(self, is_datasets_previewed: bool = False): + def deploy(self, are_datasets_previewable: bool = False): """Create and deploy all static files to local/remote file system""" - self._upload_api_responses(is_datasets_previewed) + self._upload_api_responses(are_datasets_previewable) self._upload_static_files(_HTML_DIR) self._upload_deploy_viz_metadata_file() diff --git a/package/kedro_viz/launchers/cli.py b/package/kedro_viz/launchers/cli.py index 37491cfce7..c1ef7b9309 100644 --- a/package/kedro_viz/launchers/cli.py +++ b/package/kedro_viz/launchers/cli.py @@ -285,7 +285,7 @@ def build(include_hooks, preview): def create_shareableviz_process( platform, - is_datasets_previewed, + are_datasets_previewable, endpoint=None, bucket_name=None, include_hooks=False, @@ -299,7 +299,7 @@ def create_shareableviz_process( target=load_and_deploy_viz, args=( platform, - is_datasets_previewed, + are_datasets_previewable, endpoint, bucket_name, include_hooks, @@ -375,7 +375,7 @@ def create_shareableviz_process( def load_and_deploy_viz( platform, - is_datasets_previewed, + are_datasets_previewable, endpoint, bucket_name, include_hooks, @@ -391,7 +391,7 @@ def load_and_deploy_viz( # Start the deployment deployer = DeployerFactory.create_deployer(platform, endpoint, bucket_name) - deployer.deploy(is_datasets_previewed) + deployer.deploy(are_datasets_previewable) except ( # pylint: disable=catching-non-exception diff --git a/package/kedro_viz/models/flowchart.py b/package/kedro_viz/models/flowchart.py index ff891bc779..d3d5526dac 100644 --- a/package/kedro_viz/models/flowchart.py +++ b/package/kedro_viz/models/flowchart.py @@ -735,7 +735,7 @@ class DataNodeMetadata(GraphNodeMetadata): data_node: DataNode = Field(..., exclude=True) - is_datasets_previewed: ClassVar[bool] = True + are_datasets_previewable: ClassVar[bool] = True type: Optional[str] = Field( default=None, validate_default=True, description="The type of the data node" @@ -779,8 +779,8 @@ def check_data_node_exists(cls, values): return values @classmethod - def set_is_datasets_previewed(cls, value: bool): - cls.is_datasets_previewed = value + def set_are_datasets_previewable(cls, value: bool): + cls.are_datasets_previewable = value @classmethod def set_data_node_and_dataset(cls, data_node): @@ -815,7 +815,7 @@ def set_preview(cls, _): if ( cls.data_node.is_preview_disabled() or not hasattr(cls.dataset, "preview") - or not cls.is_datasets_previewed + or not cls.are_datasets_previewable ): return None @@ -839,7 +839,11 @@ def set_preview(cls, _): @field_validator("preview_type") @classmethod def set_preview_type(cls, _): - if cls.data_node.is_preview_disabled() or not hasattr(cls.dataset, "preview"): + if ( + cls.data_node.is_preview_disabled() + or not hasattr(cls.dataset, "preview") + or not cls.are_datasets_previewable + ): return None try: diff --git a/package/tests/conftest.py b/package/tests/conftest.py index cc77364d74..b280938354 100644 --- a/package/tests/conftest.py +++ b/package/tests/conftest.py @@ -17,7 +17,7 @@ from kedro_viz.data_access import DataAccessManager from kedro_viz.integrations.kedro.hooks import DatasetStatsHook from kedro_viz.integrations.kedro.sqlite_store import SQLiteStore -from kedro_viz.models.flowchart import GraphNode +from kedro_viz.models.flowchart import DataNodeMetadata, GraphNode from kedro_viz.server import populate_data @@ -350,3 +350,8 @@ def pipeline_with_data_sets_mock(): pipeline = mock.MagicMock() pipeline.data_sets.return_value = ["model_inputs#csv"] return pipeline + + +@pytest.fixture(autouse=True) +def reset_are_datasets_previewable(): + DataNodeMetadata.are_datasets_previewable = True diff --git a/package/tests/test_api/test_rest/test_responses.py b/package/tests/test_api/test_rest/test_responses.py index 7464efefd9..d0c393ad2b 100644 --- a/package/tests/test_api/test_rest/test_responses.py +++ b/package/tests/test_api/test_rest/test_responses.py @@ -1031,7 +1031,7 @@ def test_save_api_pipeline_response_to_fs(self, mocker): mock_write_api_response_to_fs.assert_has_calls(expected_calls, any_order=True) @pytest.mark.parametrize( - "file_path, protocol, is_datasets_previewed", + "file_path, protocol, are_datasets_previewable", [ ("s3://shareableviz", "s3", True), ("abfs://shareableviz", "abfs", False), @@ -1039,7 +1039,7 @@ def test_save_api_pipeline_response_to_fs(self, mocker): ], ) def test_save_api_responses_to_fs( - self, file_path, protocol, is_datasets_previewed, mocker + self, file_path, protocol, are_datasets_previewable, mocker ): mock_api_main_response_to_fs = mocker.patch( "kedro_viz.api.rest.responses.save_api_main_response_to_fs" @@ -1055,7 +1055,7 @@ def test_save_api_responses_to_fs( mock_filesystem.return_value.protocol = protocol save_api_responses_to_fs( - file_path, mock_filesystem.return_value, is_datasets_previewed + file_path, mock_filesystem.return_value, are_datasets_previewable ) mock_api_main_response_to_fs.assert_called_once_with( @@ -1064,7 +1064,7 @@ def test_save_api_responses_to_fs( mock_api_node_response_to_fs.assert_called_once_with( f"{file_path}/api/nodes", mock_filesystem.return_value, - is_datasets_previewed, + are_datasets_previewable, ) mock_api_pipeline_response_to_fs.assert_called_once_with( f"{file_path}/api/pipelines", mock_filesystem.return_value diff --git a/package/tests/test_integrations/test_azure_deployer.py b/package/tests/test_integrations/test_azure_deployer.py index 4c524811b4..4b1a8d7a30 100644 --- a/package/tests/test_integrations/test_azure_deployer.py +++ b/package/tests/test_integrations/test_azure_deployer.py @@ -26,7 +26,7 @@ def mock_file_system(mocker): class TestAzureDeployer: - def test_deploy(self, endpoint, bucket_name, mocker): + def test_deploy(self, endpoint, bucket_name, mocker, mock_file_system): deployer = AzureDeployer(endpoint, bucket_name) mocker.patch.object(deployer, "_upload_api_responses") diff --git a/package/tests/test_integrations/test_deployer_factory.py b/package/tests/test_integrations/test_deployer_factory.py index c14794b087..638fe94065 100644 --- a/package/tests/test_integrations/test_deployer_factory.py +++ b/package/tests/test_integrations/test_deployer_factory.py @@ -18,7 +18,8 @@ ("gcp", "http://mocked-url.com", "shareableviz", GCPDeployer), ], ) -def test_create_deployer(platform, endpoint, bucket_name, deployer_class): +def test_create_deployer(platform, endpoint, bucket_name, deployer_class, mocker): + mocker.patch("fsspec.filesystem") deployer = DeployerFactory.create_deployer(platform, endpoint, bucket_name) assert isinstance(deployer, deployer_class) assert deployer._endpoint == endpoint diff --git a/package/tests/test_integrations/test_gcp_deployer.py b/package/tests/test_integrations/test_gcp_deployer.py index 56daac7747..86f4f2094e 100755 --- a/package/tests/test_integrations/test_gcp_deployer.py +++ b/package/tests/test_integrations/test_gcp_deployer.py @@ -21,7 +21,7 @@ def mock_file_system(mocker): class TestGCPDeployer: - def test_deploy(self, endpoint, bucket_name, mocker): + def test_deploy(self, endpoint, bucket_name, mocker, mock_file_system): deployer = GCPDeployer(endpoint, bucket_name) mocker.patch.object(deployer, "_upload_api_responses") diff --git a/package/tests/test_launchers/test_cli.py b/package/tests/test_launchers/test_cli.py index 7ff37d9a5a..cc3b2cf780 100755 --- a/package/tests/test_launchers/test_cli.py +++ b/package/tests/test_launchers/test_cli.py @@ -588,7 +588,7 @@ def test_successful_build_with_existing_static_files( @pytest.mark.parametrize( - "platform, is_datasets_previewed, endpoint, bucket_name," + "platform, are_datasets_previewable, endpoint, bucket_name," "include_hooks, process_completed_value", [ ( @@ -645,7 +645,7 @@ def test_successful_build_with_existing_static_files( ) def test_create_shareableviz_process( platform, - is_datasets_previewed, + are_datasets_previewable, endpoint, bucket_name, include_hooks, @@ -659,7 +659,7 @@ def test_create_shareableviz_process( ): mock_process_completed.return_value.value = process_completed_value cli.create_shareableviz_process( - platform, is_datasets_previewed, endpoint, bucket_name, include_hooks + platform, are_datasets_previewable, endpoint, bucket_name, include_hooks ) # Assert the mocks were called as expected @@ -667,7 +667,7 @@ def test_create_shareableviz_process( target=mock_viz_load_and_deploy, args=( platform, - is_datasets_previewed, + are_datasets_previewable, endpoint, bucket_name, include_hooks, @@ -707,7 +707,7 @@ def test_create_shareableviz_process( @pytest.mark.parametrize( - "platform, is_datasets_previewed, endpoint, bucket_name, include_hooks, package_name", + "platform, are_datasets_previewable, endpoint, bucket_name, include_hooks, package_name", [ ( "azure", @@ -731,7 +731,7 @@ def test_create_shareableviz_process( ) def test_load_and_deploy_viz_success( platform, - is_datasets_previewed, + are_datasets_previewable, endpoint, bucket_name, include_hooks, @@ -747,7 +747,7 @@ def test_load_and_deploy_viz_success( cli.load_and_deploy_viz( platform, - is_datasets_previewed, + are_datasets_previewable, endpoint, bucket_name, include_hooks, @@ -762,5 +762,5 @@ def test_load_and_deploy_viz_success( mock_DeployerFactory.create_deployer.assert_called_once_with( platform, endpoint, bucket_name ) - deployer_mock.deploy.assert_called_once_with(is_datasets_previewed) + deployer_mock.deploy.assert_called_once_with(are_datasets_previewable) mock_click_echo.echo.assert_not_called() diff --git a/package/tests/test_models/test_flowchart.py b/package/tests/test_models/test_flowchart.py index a81e4c45c1..f578d0b140 100644 --- a/package/tests/test_models/test_flowchart.py +++ b/package/tests/test_models/test_flowchart.py @@ -394,6 +394,13 @@ def test_is_preview_disabled(self): ) assert data_node.is_preview_disabled() is True + def test_are_datasets_previewable(self, example_data_node): + DataNodeMetadata.set_are_datasets_previewable(False) + preview_node_metadata = DataNodeMetadata(data_node=example_data_node) + + assert preview_node_metadata.preview is None + assert preview_node_metadata.preview_type is None + def test_preview_data_node_metadata(self, example_data_node): expected_preview_data = { "columns": ["id", "company_rating", "company_location"], @@ -405,10 +412,10 @@ def test_preview_data_node_metadata(self, example_data_node): ], } - DataNodeMetadata.set_is_datasets_previewed(True) preview_node_metadata = DataNodeMetadata(data_node=example_data_node) assert preview_node_metadata.preview == expected_preview_data + assert preview_node_metadata.preview_type == "TablePreview" def test_preview_data_node_metadata_exception(self, caplog): empty_dataset = CSVDataset(filepath="temp.csv") From 57b68b3922c10b4b22e3d7f8565084434ccd9bcf Mon Sep 17 00:00:00 2001 From: ravi-kumar-pilla Date: Thu, 16 May 2024 23:06:20 -0500 Subject: [PATCH 23/35] adjust file permission Signed-off-by: ravi-kumar-pilla --- package/tests/test_integrations/test_base_deployer.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 package/tests/test_integrations/test_base_deployer.py diff --git a/package/tests/test_integrations/test_base_deployer.py b/package/tests/test_integrations/test_base_deployer.py old mode 100755 new mode 100644 From 50ef2a4749300f7bf419cfca5ef69835810c8ac8 Mon Sep 17 00:00:00 2001 From: ravi-kumar-pilla Date: Thu, 16 May 2024 23:34:20 -0500 Subject: [PATCH 24/35] update tests Signed-off-by: ravi-kumar-pilla --- package/kedro_viz/api/rest/requests.py | 2 +- package/kedro_viz/api/rest/router.py | 2 +- .../tests/test_api/test_rest/test_router.py | 19 +++++++++++++------ 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/package/kedro_viz/api/rest/requests.py b/package/kedro_viz/api/rest/requests.py index a6ce949d3d..e499188b29 100644 --- a/package/kedro_viz/api/rest/requests.py +++ b/package/kedro_viz/api/rest/requests.py @@ -7,6 +7,6 @@ class DeployerConfiguration(BaseModel): """Credentials for Deployers.""" platform: str - is_datasets_previewed: bool + are_datasets_previewable: bool = False endpoint: str bucket_name: str diff --git a/package/kedro_viz/api/rest/router.py b/package/kedro_viz/api/rest/router.py index c72003eb60..c21fa8b81f 100644 --- a/package/kedro_viz/api/rest/router.py +++ b/package/kedro_viz/api/rest/router.py @@ -63,7 +63,7 @@ async def deploy_kedro_viz(input_values: DeployerConfiguration): deployer = DeployerFactory.create_deployer( input_values.platform, input_values.endpoint, input_values.bucket_name ) - deployer.deploy(input_values.is_datasets_previewed) + deployer.deploy(input_values.are_datasets_previewable) response = { "message": "Website deployed on " f"{input_values.platform and input_values.platform.upper()}", diff --git a/package/tests/test_api/test_rest/test_router.py b/package/tests/test_api/test_rest/test_router.py index 9b5c28cc18..ae83ca23a0 100644 --- a/package/tests/test_api/test_rest/test_router.py +++ b/package/tests/test_api/test_rest/test_router.py @@ -6,25 +6,32 @@ class MockDeployer: def __init__(self, platform, endpoint, bucket_name): pass - def deploy(self): + def deploy(self, are_datasets_previewable): pass @pytest.mark.parametrize( - "platform, endpoint, bucket_name", + "platform, endpoint, bucket_name, are_datasets_previewable", [ - ("aws", "http://mocked-url.com", "s3://shareableviz"), - ("azure", "http://mocked-url.com", "abfs://shareableviz"), + ("aws", "http://mocked-url.com", "s3://shareableviz", True), + ("azure", "http://mocked-url.com", "abfs://shareableviz", False), ], ) -def test_deploy_kedro_viz(client, platform, endpoint, bucket_name, mocker): +def test_deploy_kedro_viz( + client, platform, endpoint, bucket_name, are_datasets_previewable, mocker +): mocker.patch( "kedro_viz.api.rest.router.DeployerFactory.create_deployer", return_value=MockDeployer(platform, endpoint, bucket_name), ) response = client.post( "/api/deploy", - json={"platform": platform, "endpoint": endpoint, "bucket_name": bucket_name}, + json={ + "platform": platform, + "endpoint": endpoint, + "bucket_name": bucket_name, + "are_datasets_previewable": are_datasets_previewable, + }, ) assert response.status_code == 200 From e523f23a682027086f5aabf371a6f8eceab2fd47 Mon Sep 17 00:00:00 2001 From: ravi-kumar-pilla Date: Thu, 16 May 2024 23:46:25 -0500 Subject: [PATCH 25/35] update release note Signed-off-by: ravi-kumar-pilla --- RELEASE.md | 3 ++- package/kedro_viz/api/rest/requests.py | 0 2 files changed, 2 insertions(+), 1 deletion(-) mode change 100644 => 100755 package/kedro_viz/api/rest/requests.py diff --git a/RELEASE.md b/RELEASE.md index 2ede3cee15..f30e1303fa 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -9,7 +9,8 @@ Please follow the established format: ## Major features and improvements -- Enable/disable preview for all the datasets when publishing Kedro-Viz with CLI. (#1894) +- Enable/disable preview for all the datasets when publishing Kedro-Viz from CLI. (#1894) +- Enable/disable preview for all the datasets when publishing Kedro-Viz from UI. (#1895) ## Bug fixes and other changes diff --git a/package/kedro_viz/api/rest/requests.py b/package/kedro_viz/api/rest/requests.py old mode 100644 new mode 100755 From 4f5c446679c7b45095588941bbcc7cf494cb9016 Mon Sep 17 00:00:00 2001 From: ravi-kumar-pilla Date: Thu, 16 May 2024 23:47:43 -0500 Subject: [PATCH 26/35] update release note Signed-off-by: ravi-kumar-pilla --- RELEASE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE.md b/RELEASE.md index 2ede3cee15..6633224477 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -9,7 +9,7 @@ Please follow the established format: ## Major features and improvements -- Enable/disable preview for all the datasets when publishing Kedro-Viz with CLI. (#1894) +- Enable/disable preview for all the datasets when publishing Kedro-Viz from CLI. (#1894) ## Bug fixes and other changes From c815a9b3fb591c9333ad9d431be3b3732d8158f8 Mon Sep 17 00:00:00 2001 From: Jitendra Gundaniya Date: Mon, 20 May 2024 19:10:55 +0100 Subject: [PATCH 27/35] --include-preview flag added for CLI Signed-off-by: Jitendra Gundaniya --- package/kedro_viz/launchers/cli.py | 16 ++++++++-------- package/tests/test_launchers/test_cli.py | 5 ++--- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/package/kedro_viz/launchers/cli.py b/package/kedro_viz/launchers/cli.py index c1ef7b9309..ca0a8953e1 100644 --- a/package/kedro_viz/launchers/cli.py +++ b/package/kedro_viz/launchers/cli.py @@ -235,11 +235,11 @@ def run( help="A flag to include all registered hooks in your Kedro Project", ) @click.option( - "--preview", - default=False, + "--include-preview", + is_flag=True, help="Enable/disable preview for all the datasets.", ) -def deploy(platform, endpoint, bucket_name, include_hooks, preview): +def deploy(platform, endpoint, bucket_name, include_hooks, include_preview): """Deploy and host Kedro Viz on provided platform""" if not platform or platform.lower() not in SHAREABLEVIZ_SUPPORTED_PLATFORMS: display_cli_message( @@ -259,7 +259,7 @@ def deploy(platform, endpoint, bucket_name, include_hooks, preview): create_shareableviz_process( platform, - preview, + include_preview, endpoint, bucket_name, include_hooks, @@ -273,14 +273,14 @@ def deploy(platform, endpoint, bucket_name, include_hooks, preview): help="A flag to include all registered hooks in your Kedro Project", ) @click.option( - "--preview", - default=False, + "--include-preview", + is_flag=True, help="Enable/disable preview for all the datasets.", ) -def build(include_hooks, preview): +def build(include_hooks, include_preview): """Create build directory of local Kedro Viz instance with Kedro project data""" - create_shareableviz_process("local", preview, include_hooks=include_hooks) + create_shareableviz_process("local", include_preview, include_hooks=include_hooks) def create_shareableviz_process( diff --git a/package/tests/test_launchers/test_cli.py b/package/tests/test_launchers/test_cli.py index cc3b2cf780..df66ebed05 100755 --- a/package/tests/test_launchers/test_cli.py +++ b/package/tests/test_launchers/test_cli.py @@ -455,8 +455,7 @@ def test_viz_command_group(mocker, mock_click_echo): "http://example-bucket.s3-website.us-east-2.amazonaws.com/", "--bucket-name", "example-bucket", - "--preview", - "true", + "--include-preview", ], { "platform": "aws", @@ -561,7 +560,7 @@ def test_viz_deploy_invalid_endpoint(mocker, mock_click_echo): {"platform": "local", "include_hooks": True}, ), ( - ["viz", "build", "--preview", "true"], + ["viz", "build", "--include-preview"], {"platform": "local", "preview": True}, ), ], From d06f36ea0448f69ae1f3e88405997099af6f2f47 Mon Sep 17 00:00:00 2001 From: Jitendra Gundaniya Date: Tue, 21 May 2024 18:53:15 +0100 Subject: [PATCH 28/35] are_datasets_previewable renamed to is_all_previews_enabled Signed-off-by: Jitendra Gundaniya --- package/kedro_viz/api/rest/responses.py | 8 ++++---- .../integrations/deployment/base_deployer.py | 8 ++++---- package/kedro_viz/launchers/cli.py | 8 ++++---- package/kedro_viz/models/flowchart.py | 19 +++++++++++++------ package/tests/conftest.py | 4 ++-- .../test_api/test_rest/test_responses.py | 8 ++++---- package/tests/test_launchers/test_cli.py | 16 ++++++++-------- package/tests/test_models/test_flowchart.py | 4 ++-- 8 files changed, 41 insertions(+), 34 deletions(-) diff --git a/package/kedro_viz/api/rest/responses.py b/package/kedro_viz/api/rest/responses.py index 2c2ce33424..8da71d0354 100644 --- a/package/kedro_viz/api/rest/responses.py +++ b/package/kedro_viz/api/rest/responses.py @@ -421,11 +421,11 @@ def save_api_main_response_to_fs(main_path: str, remote_fs: Any): def save_api_node_response_to_fs( - nodes_path: str, remote_fs: Any, are_datasets_previewable: bool + nodes_path: str, remote_fs: Any, is_all_previews_enabled: bool ): """Saves API /nodes/{node} response to a directory.""" # Set if preview is enabled/disabled for all data nodes - DataNodeMetadata.set_are_datasets_previewable(are_datasets_previewable) + DataNodeMetadata.set_is_all_previews_enabled(is_all_previews_enabled) for nodeId in data_access_manager.nodes.get_node_ids(): try: @@ -457,7 +457,7 @@ def save_api_pipeline_response_to_fs(pipelines_path: str, remote_fs: Any): raise exc -def save_api_responses_to_fs(path: str, remote_fs: Any, are_datasets_previewable: bool): +def save_api_responses_to_fs(path: str, remote_fs: Any, is_all_previews_enabled: bool): """Saves all Kedro Viz API responses to a directory.""" try: logger.debug( @@ -475,7 +475,7 @@ def save_api_responses_to_fs(path: str, remote_fs: Any, are_datasets_previewable remote_fs.makedirs(pipelines_path, exist_ok=True) save_api_main_response_to_fs(main_path, remote_fs) - save_api_node_response_to_fs(nodes_path, remote_fs, are_datasets_previewable) + save_api_node_response_to_fs(nodes_path, remote_fs, is_all_previews_enabled) save_api_pipeline_response_to_fs(pipelines_path, remote_fs) except Exception as exc: # pragma: no cover diff --git a/package/kedro_viz/integrations/deployment/base_deployer.py b/package/kedro_viz/integrations/deployment/base_deployer.py index 73e23d815a..35b7fc1818 100644 --- a/package/kedro_viz/integrations/deployment/base_deployer.py +++ b/package/kedro_viz/integrations/deployment/base_deployer.py @@ -33,9 +33,9 @@ def __init__(self): self._path = None self._fs = None - def _upload_api_responses(self, are_datasets_previewable: bool): + def _upload_api_responses(self, is_all_previews_enabled: bool): """Write API responses to the build.""" - save_api_responses_to_fs(self._path, self._fs, are_datasets_previewable) + save_api_responses_to_fs(self._path, self._fs, is_all_previews_enabled) def _ingest_heap_analytics(self): """Ingest heap analytics to index file in the build.""" @@ -98,9 +98,9 @@ def _upload_deploy_viz_metadata_file(self): logger.exception("Upload failed: %s ", exc) raise exc - def deploy(self, are_datasets_previewable: bool = False): + def deploy(self, is_all_previews_enabled: bool = False): """Create and deploy all static files to local/remote file system""" - self._upload_api_responses(are_datasets_previewable) + self._upload_api_responses(is_all_previews_enabled) self._upload_static_files(_HTML_DIR) self._upload_deploy_viz_metadata_file() diff --git a/package/kedro_viz/launchers/cli.py b/package/kedro_viz/launchers/cli.py index ca0a8953e1..b3bf7654bc 100644 --- a/package/kedro_viz/launchers/cli.py +++ b/package/kedro_viz/launchers/cli.py @@ -285,7 +285,7 @@ def build(include_hooks, include_preview): def create_shareableviz_process( platform, - are_datasets_previewable, + is_all_previews_enabled, endpoint=None, bucket_name=None, include_hooks=False, @@ -299,7 +299,7 @@ def create_shareableviz_process( target=load_and_deploy_viz, args=( platform, - are_datasets_previewable, + is_all_previews_enabled, endpoint, bucket_name, include_hooks, @@ -375,7 +375,7 @@ def create_shareableviz_process( def load_and_deploy_viz( platform, - are_datasets_previewable, + is_all_previews_enabled, endpoint, bucket_name, include_hooks, @@ -391,7 +391,7 @@ def load_and_deploy_viz( # Start the deployment deployer = DeployerFactory.create_deployer(platform, endpoint, bucket_name) - deployer.deploy(are_datasets_previewable) + deployer.deploy(is_all_previews_enabled) except ( # pylint: disable=catching-non-exception diff --git a/package/kedro_viz/models/flowchart.py b/package/kedro_viz/models/flowchart.py index d3d5526dac..813e68c142 100644 --- a/package/kedro_viz/models/flowchart.py +++ b/package/kedro_viz/models/flowchart.py @@ -646,7 +646,10 @@ def get_preview_args(self): return self.viz_metadata.get("preview_args", None) def is_preview_disabled(self): - """Checks if the dataset has a preview disabled""" + """Checks if the dataset has a preview disabled at the node level. + This happens when the user specifies 'preview=false' in the DataCatalog. + """ + return ( self.viz_metadata is not None and self.viz_metadata.get("preview") is False ) @@ -729,13 +732,17 @@ class DataNodeMetadata(GraphNodeMetadata): Args: data_node (DataNode): Data node to which this metadata belongs to. + Attributes: + is_all_previews_enabled (bool): Class-level attribute to determine if previews are enabled for all nodes. + This can be configured via CLI or UI to manage the preview settings. + Raises: AssertionError: If data_node is not supplied during instantiation """ data_node: DataNode = Field(..., exclude=True) - are_datasets_previewable: ClassVar[bool] = True + is_all_previews_enabled: ClassVar[bool] = True type: Optional[str] = Field( default=None, validate_default=True, description="The type of the data node" @@ -779,8 +786,8 @@ def check_data_node_exists(cls, values): return values @classmethod - def set_are_datasets_previewable(cls, value: bool): - cls.are_datasets_previewable = value + def set_is_all_previews_enabled(cls, value: bool): + cls.is_all_previews_enabled = value @classmethod def set_data_node_and_dataset(cls, data_node): @@ -815,7 +822,7 @@ def set_preview(cls, _): if ( cls.data_node.is_preview_disabled() or not hasattr(cls.dataset, "preview") - or not cls.are_datasets_previewable + or not cls.is_all_previews_enabled ): return None @@ -842,7 +849,7 @@ def set_preview_type(cls, _): if ( cls.data_node.is_preview_disabled() or not hasattr(cls.dataset, "preview") - or not cls.are_datasets_previewable + or not cls.is_all_previews_enabled ): return None diff --git a/package/tests/conftest.py b/package/tests/conftest.py index b280938354..2c6426105a 100644 --- a/package/tests/conftest.py +++ b/package/tests/conftest.py @@ -353,5 +353,5 @@ def pipeline_with_data_sets_mock(): @pytest.fixture(autouse=True) -def reset_are_datasets_previewable(): - DataNodeMetadata.are_datasets_previewable = True +def reset_is_all_previews_enabled(): + DataNodeMetadata.is_all_previews_enabled = True diff --git a/package/tests/test_api/test_rest/test_responses.py b/package/tests/test_api/test_rest/test_responses.py index d0c393ad2b..da46a7a5f3 100644 --- a/package/tests/test_api/test_rest/test_responses.py +++ b/package/tests/test_api/test_rest/test_responses.py @@ -1031,7 +1031,7 @@ def test_save_api_pipeline_response_to_fs(self, mocker): mock_write_api_response_to_fs.assert_has_calls(expected_calls, any_order=True) @pytest.mark.parametrize( - "file_path, protocol, are_datasets_previewable", + "file_path, protocol, is_all_previews_enabled", [ ("s3://shareableviz", "s3", True), ("abfs://shareableviz", "abfs", False), @@ -1039,7 +1039,7 @@ def test_save_api_pipeline_response_to_fs(self, mocker): ], ) def test_save_api_responses_to_fs( - self, file_path, protocol, are_datasets_previewable, mocker + self, file_path, protocol, is_all_previews_enabled, mocker ): mock_api_main_response_to_fs = mocker.patch( "kedro_viz.api.rest.responses.save_api_main_response_to_fs" @@ -1055,7 +1055,7 @@ def test_save_api_responses_to_fs( mock_filesystem.return_value.protocol = protocol save_api_responses_to_fs( - file_path, mock_filesystem.return_value, are_datasets_previewable + file_path, mock_filesystem.return_value, is_all_previews_enabled ) mock_api_main_response_to_fs.assert_called_once_with( @@ -1064,7 +1064,7 @@ def test_save_api_responses_to_fs( mock_api_node_response_to_fs.assert_called_once_with( f"{file_path}/api/nodes", mock_filesystem.return_value, - are_datasets_previewable, + is_all_previews_enabled, ) mock_api_pipeline_response_to_fs.assert_called_once_with( f"{file_path}/api/pipelines", mock_filesystem.return_value diff --git a/package/tests/test_launchers/test_cli.py b/package/tests/test_launchers/test_cli.py index df66ebed05..94c75c95a3 100755 --- a/package/tests/test_launchers/test_cli.py +++ b/package/tests/test_launchers/test_cli.py @@ -587,7 +587,7 @@ def test_successful_build_with_existing_static_files( @pytest.mark.parametrize( - "platform, are_datasets_previewable, endpoint, bucket_name," + "platform, is_all_previews_enabled, endpoint, bucket_name," "include_hooks, process_completed_value", [ ( @@ -644,7 +644,7 @@ def test_successful_build_with_existing_static_files( ) def test_create_shareableviz_process( platform, - are_datasets_previewable, + is_all_previews_enabled, endpoint, bucket_name, include_hooks, @@ -658,7 +658,7 @@ def test_create_shareableviz_process( ): mock_process_completed.return_value.value = process_completed_value cli.create_shareableviz_process( - platform, are_datasets_previewable, endpoint, bucket_name, include_hooks + platform, is_all_previews_enabled, endpoint, bucket_name, include_hooks ) # Assert the mocks were called as expected @@ -666,7 +666,7 @@ def test_create_shareableviz_process( target=mock_viz_load_and_deploy, args=( platform, - are_datasets_previewable, + is_all_previews_enabled, endpoint, bucket_name, include_hooks, @@ -706,7 +706,7 @@ def test_create_shareableviz_process( @pytest.mark.parametrize( - "platform, are_datasets_previewable, endpoint, bucket_name, include_hooks, package_name", + "platform, is_all_previews_enabled, endpoint, bucket_name, include_hooks, package_name", [ ( "azure", @@ -730,7 +730,7 @@ def test_create_shareableviz_process( ) def test_load_and_deploy_viz_success( platform, - are_datasets_previewable, + is_all_previews_enabled, endpoint, bucket_name, include_hooks, @@ -746,7 +746,7 @@ def test_load_and_deploy_viz_success( cli.load_and_deploy_viz( platform, - are_datasets_previewable, + is_all_previews_enabled, endpoint, bucket_name, include_hooks, @@ -761,5 +761,5 @@ def test_load_and_deploy_viz_success( mock_DeployerFactory.create_deployer.assert_called_once_with( platform, endpoint, bucket_name ) - deployer_mock.deploy.assert_called_once_with(are_datasets_previewable) + deployer_mock.deploy.assert_called_once_with(is_all_previews_enabled) mock_click_echo.echo.assert_not_called() diff --git a/package/tests/test_models/test_flowchart.py b/package/tests/test_models/test_flowchart.py index f578d0b140..5531861f17 100644 --- a/package/tests/test_models/test_flowchart.py +++ b/package/tests/test_models/test_flowchart.py @@ -394,8 +394,8 @@ def test_is_preview_disabled(self): ) assert data_node.is_preview_disabled() is True - def test_are_datasets_previewable(self, example_data_node): - DataNodeMetadata.set_are_datasets_previewable(False) + def test_is_all_previews_enabled(self, example_data_node): + DataNodeMetadata.set_is_all_previews_enabled(False) preview_node_metadata = DataNodeMetadata(data_node=example_data_node) assert preview_node_metadata.preview is None From 73b1944f0cedbc656ae2bffd58480ae4ea81faef Mon Sep 17 00:00:00 2001 From: Jitendra Gundaniya Date: Tue, 21 May 2024 19:04:05 +0100 Subject: [PATCH 29/35] lint fix Signed-off-by: Jitendra Gundaniya --- package/kedro_viz/models/flowchart.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/package/kedro_viz/models/flowchart.py b/package/kedro_viz/models/flowchart.py index 813e68c142..dbb842af3e 100644 --- a/package/kedro_viz/models/flowchart.py +++ b/package/kedro_viz/models/flowchart.py @@ -733,8 +733,9 @@ class DataNodeMetadata(GraphNodeMetadata): data_node (DataNode): Data node to which this metadata belongs to. Attributes: - is_all_previews_enabled (bool): Class-level attribute to determine if previews are enabled for all nodes. - This can be configured via CLI or UI to manage the preview settings. + is_all_previews_enabled (bool): Class-level attribute to determine if + previews are enabled for all nodes. This can be configured via CLI + or UI to manage the preview settings. Raises: AssertionError: If data_node is not supplied during instantiation From 2294cc6a2d42b7a372dee0c7d6305766b22e002d Mon Sep 17 00:00:00 2001 From: Jitendra Gundaniya Date: Tue, 21 May 2024 19:07:26 +0100 Subject: [PATCH 30/35] are_datasets_previewable renamed to is_all_previews_enabled Signed-off-by: Jitendra Gundaniya --- package/kedro_viz/api/rest/requests.py | 2 +- package/kedro_viz/api/rest/router.py | 2 +- package/tests/test_api/test_rest/test_router.py | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/package/kedro_viz/api/rest/requests.py b/package/kedro_viz/api/rest/requests.py index e499188b29..6f0a8bafb3 100755 --- a/package/kedro_viz/api/rest/requests.py +++ b/package/kedro_viz/api/rest/requests.py @@ -7,6 +7,6 @@ class DeployerConfiguration(BaseModel): """Credentials for Deployers.""" platform: str - are_datasets_previewable: bool = False + is_all_previews_enabled: bool = False endpoint: str bucket_name: str diff --git a/package/kedro_viz/api/rest/router.py b/package/kedro_viz/api/rest/router.py index c21fa8b81f..d8a8499677 100644 --- a/package/kedro_viz/api/rest/router.py +++ b/package/kedro_viz/api/rest/router.py @@ -63,7 +63,7 @@ async def deploy_kedro_viz(input_values: DeployerConfiguration): deployer = DeployerFactory.create_deployer( input_values.platform, input_values.endpoint, input_values.bucket_name ) - deployer.deploy(input_values.are_datasets_previewable) + deployer.deploy(input_values.is_all_previews_enabled) response = { "message": "Website deployed on " f"{input_values.platform and input_values.platform.upper()}", diff --git a/package/tests/test_api/test_rest/test_router.py b/package/tests/test_api/test_rest/test_router.py index ae83ca23a0..9ba1de7e2c 100644 --- a/package/tests/test_api/test_rest/test_router.py +++ b/package/tests/test_api/test_rest/test_router.py @@ -6,19 +6,19 @@ class MockDeployer: def __init__(self, platform, endpoint, bucket_name): pass - def deploy(self, are_datasets_previewable): + def deploy(self, is_all_previews_enabled): pass @pytest.mark.parametrize( - "platform, endpoint, bucket_name, are_datasets_previewable", + "platform, endpoint, bucket_name, is_all_previews_enabled", [ ("aws", "http://mocked-url.com", "s3://shareableviz", True), ("azure", "http://mocked-url.com", "abfs://shareableviz", False), ], ) def test_deploy_kedro_viz( - client, platform, endpoint, bucket_name, are_datasets_previewable, mocker + client, platform, endpoint, bucket_name, is_all_previews_enabled, mocker ): mocker.patch( "kedro_viz.api.rest.router.DeployerFactory.create_deployer", @@ -30,7 +30,7 @@ def test_deploy_kedro_viz( "platform": platform, "endpoint": endpoint, "bucket_name": bucket_name, - "are_datasets_previewable": are_datasets_previewable, + "is_all_previews_enabled": is_all_previews_enabled, }, ) From 94e3501a0e46811f199f500aec1862b9070a6f8d Mon Sep 17 00:00:00 2001 From: Jitendra Gundaniya Date: Tue, 21 May 2024 20:50:34 +0100 Subject: [PATCH 31/35] type ignore add for TRANSCODING_SEPARATOR Signed-off-by: Jitendra Gundaniya --- package/kedro_viz/data_access/repositories/catalog.py | 2 +- package/kedro_viz/integrations/kedro/hooks.py | 5 ++++- package/kedro_viz/models/flowchart.py | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/package/kedro_viz/data_access/repositories/catalog.py b/package/kedro_viz/data_access/repositories/catalog.py index 427a13cf0d..bfbd61a437 100644 --- a/package/kedro_viz/data_access/repositories/catalog.py +++ b/package/kedro_viz/data_access/repositories/catalog.py @@ -12,7 +12,7 @@ from kedro.pipeline._transcoding import TRANSCODING_SEPARATOR, _strip_transcoding except ImportError: # pragma: no cover # older versions - from kedro.pipeline.pipeline import TRANSCODING_SEPARATOR, _strip_transcoding + from kedro.pipeline.pipeline import TRANSCODING_SEPARATOR, _strip_transcoding # type: ignore from packaging.version import parse diff --git a/package/kedro_viz/integrations/kedro/hooks.py b/package/kedro_viz/integrations/kedro/hooks.py index d68c5eea70..32f2ffa594 100644 --- a/package/kedro_viz/integrations/kedro/hooks.py +++ b/package/kedro_viz/integrations/kedro/hooks.py @@ -17,7 +17,10 @@ from kedro.pipeline._transcoding import TRANSCODING_SEPARATOR, _strip_transcoding except ImportError: # pragma: no cover # older versions - from kedro.pipeline.pipeline import TRANSCODING_SEPARATOR, _strip_transcoding + from kedro.pipeline.pipeline import ( # type: ignore + TRANSCODING_SEPARATOR, + _strip_transcoding, + ) logger = logging.getLogger(__name__) diff --git a/package/kedro_viz/models/flowchart.py b/package/kedro_viz/models/flowchart.py index dbb842af3e..00c0525c1d 100644 --- a/package/kedro_viz/models/flowchart.py +++ b/package/kedro_viz/models/flowchart.py @@ -17,7 +17,7 @@ from kedro.pipeline._transcoding import TRANSCODING_SEPARATOR, _strip_transcoding except ImportError: # pragma: no cover # older versions - from kedro.pipeline.pipeline import TRANSCODING_SEPARATOR, _strip_transcoding + from kedro.pipeline.pipeline import TRANSCODING_SEPARATOR, _strip_transcoding # type: ignore from pydantic import ( BaseModel, From 6f088b5e0d8ca4e28387a7b6c277730252035460 Mon Sep 17 00:00:00 2001 From: Jitendra Gundaniya Date: Wed, 29 May 2024 20:32:49 +0100 Subject: [PATCH 32/35] File permissions updated Signed-off-by: Jitendra Gundaniya --- package/kedro_viz/api/rest/requests.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 package/kedro_viz/api/rest/requests.py diff --git a/package/kedro_viz/api/rest/requests.py b/package/kedro_viz/api/rest/requests.py old mode 100755 new mode 100644 From c706625ebcca393a59db086296d2dab4927b27d6 Mon Sep 17 00:00:00 2001 From: Jitendra Gundaniya Date: Fri, 31 May 2024 14:12:52 +0100 Subject: [PATCH 33/35] fix Signed-off-by: Jitendra Gundaniya --- RELEASE.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/RELEASE.md b/RELEASE.md index e2dea492da..e8784d5cf5 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -19,9 +19,6 @@ Please follow the established format: - Introduce the toggle to expand and collapse all pipelines button in the utility bar. (#1858) - Allow Kedro-Viz commands to run from any sub directory within Kedro project. (#1871) -- Enable/disable preview for all the datasets when publishing Kedro-Viz from CLI. (#1894) -- Enable/disable preview for all the datasets when publishing Kedro-Viz from UI. (#1895) - ## Bug fixes and other changes - Fix broken URL when active pipeline name changes on initial load. (#1914) - Fix bug related to tag filtering and sharing with stateful URL. (#1878) From 9d50abba7c56cf93247d8e0d4cad77760cce5f4f Mon Sep 17 00:00:00 2001 From: Jitendra Gundaniya Date: Fri, 7 Jun 2024 13:20:10 +0100 Subject: [PATCH 34/35] All dataset preview toggle works with deploy api Signed-off-by: Jitendra Gundaniya --- src/components/shareable-url-modal/shareable-url-modal.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/components/shareable-url-modal/shareable-url-modal.js b/src/components/shareable-url-modal/shareable-url-modal.js index f65cdd07af..19261561c8 100644 --- a/src/components/shareable-url-modal/shareable-url-modal.js +++ b/src/components/shareable-url-modal/shareable-url-modal.js @@ -43,7 +43,7 @@ const ShareableUrlModal = ({ onToggleModal, visible }) => { const [hostingPlatformLocalStorageVal, setHostingPlatformLocalStorageVal] = useState(loadLocalStorage(localStorageShareableUrl) || {}); const [publishedPlatformKey, setPublishedPlatformKey] = useState(undefined); - const [isPreviewEnabled, setIsPreviewEnabled] = useState(true); + const [isPreviewEnabled, setIsPreviewEnabled] = useState(false); useEffect(() => { async function fetchPackageCompatibility() { @@ -171,7 +171,10 @@ const ShareableUrlModal = ({ onToggleModal, visible }) => { setShowPublishedView(false); try { - const request = await deployViz(inputValues); + const request = await deployViz({ + ...inputValues, + is_all_previews_enabled: isPreviewEnabled, + }); const response = await request.json(); if (request.ok) { @@ -223,6 +226,7 @@ const ShareableUrlModal = ({ onToggleModal, visible }) => { hasPlatform: false, hasEndpoint: false, }); + setIsPreviewEnabled(false); }; const { platform } = inputValues || {}; From 4a82230de28b77cc1e0a2555c2c8001c5a937b13 Mon Sep 17 00:00:00 2001 From: Jitendra Gundaniya Date: Tue, 11 Jun 2024 17:44:34 +0100 Subject: [PATCH 35/35] CLI flag renamed to --include-previews Signed-off-by: Jitendra Gundaniya --- package/kedro_viz/launchers/cli.py | 16 ++++++++-------- package/tests/test_launchers/test_cli.py | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/package/kedro_viz/launchers/cli.py b/package/kedro_viz/launchers/cli.py index b3bf7654bc..daad1e2b8e 100644 --- a/package/kedro_viz/launchers/cli.py +++ b/package/kedro_viz/launchers/cli.py @@ -235,11 +235,11 @@ def run( help="A flag to include all registered hooks in your Kedro Project", ) @click.option( - "--include-preview", + "--include-previews", is_flag=True, - help="Enable/disable preview for all the datasets.", + help="A flag to include preview for all the datasets", ) -def deploy(platform, endpoint, bucket_name, include_hooks, include_preview): +def deploy(platform, endpoint, bucket_name, include_hooks, include_previews): """Deploy and host Kedro Viz on provided platform""" if not platform or platform.lower() not in SHAREABLEVIZ_SUPPORTED_PLATFORMS: display_cli_message( @@ -259,7 +259,7 @@ def deploy(platform, endpoint, bucket_name, include_hooks, include_preview): create_shareableviz_process( platform, - include_preview, + include_previews, endpoint, bucket_name, include_hooks, @@ -273,14 +273,14 @@ def deploy(platform, endpoint, bucket_name, include_hooks, include_preview): help="A flag to include all registered hooks in your Kedro Project", ) @click.option( - "--include-preview", + "--include-previews", is_flag=True, - help="Enable/disable preview for all the datasets.", + help="A flag to include preview for all the datasets", ) -def build(include_hooks, include_preview): +def build(include_hooks, include_previews): """Create build directory of local Kedro Viz instance with Kedro project data""" - create_shareableviz_process("local", include_preview, include_hooks=include_hooks) + create_shareableviz_process("local", include_previews, include_hooks=include_hooks) def create_shareableviz_process( diff --git a/package/tests/test_launchers/test_cli.py b/package/tests/test_launchers/test_cli.py index 94c75c95a3..d30e651bba 100755 --- a/package/tests/test_launchers/test_cli.py +++ b/package/tests/test_launchers/test_cli.py @@ -455,7 +455,7 @@ def test_viz_command_group(mocker, mock_click_echo): "http://example-bucket.s3-website.us-east-2.amazonaws.com/", "--bucket-name", "example-bucket", - "--include-preview", + "--include-previews", ], { "platform": "aws", @@ -560,7 +560,7 @@ def test_viz_deploy_invalid_endpoint(mocker, mock_click_echo): {"platform": "local", "include_hooks": True}, ), ( - ["viz", "build", "--include-preview"], + ["viz", "build", "--include-previews"], {"platform": "local", "preview": True}, ), ],