From 4a4e22a59472a630e1f78842844d8c4268fd4262 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 1 Nov 2024 10:53:02 +0000 Subject: [PATCH 1/5] Bump kedro-sphinx-theme from 2024.4.0 to 2024.10.2 in /package (#2168) --- package/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package/pyproject.toml b/package/pyproject.toml index 7c39412920..3b9c0bab49 100644 --- a/package/pyproject.toml +++ b/package/pyproject.toml @@ -29,7 +29,7 @@ Tracker = "https://github.com/kedro-org/kedro-viz/issues" [project.optional-dependencies] docs = [ - "kedro-sphinx-theme==2024.4.0", + "kedro-sphinx-theme==2024.10.2", ] aws = ["s3fs>=2021.4"] azure = ["adlfs>=2021.4"] From c7cdab95172fe6fe60c3e50b34389b81c46bd697 Mon Sep 17 00:00:00 2001 From: rashidakanchwala <37628668+rashidakanchwala@users.noreply.github.com> Date: Fri, 1 Nov 2024 12:40:02 +0000 Subject: [PATCH 2/5] Fix `tag` being undefined bug from the backend. (#2162) Resolves #2106 --- .../data_access/repositories/graph.py | 9 ++-- package/tests/conftest.py | 53 +++++++++++++++++++ .../test_responses/test_pipelines.py | 14 +++++ 3 files changed, 72 insertions(+), 4 deletions(-) diff --git a/package/kedro_viz/data_access/repositories/graph.py b/package/kedro_viz/data_access/repositories/graph.py index bea6095bc9..463012800b 100644 --- a/package/kedro_viz/data_access/repositories/graph.py +++ b/package/kedro_viz/data_access/repositories/graph.py @@ -12,11 +12,12 @@ def __init__(self): self.nodes_dict: Dict[str, GraphNode] = {} self.nodes_list: List[GraphNode] = [] - def has_node(self, node: GraphNode) -> bool: - return node.id in self.nodes_dict - def add_node(self, node: GraphNode) -> GraphNode: - if not self.has_node(node): + existing_node = self.nodes_dict.get(node.id) + if existing_node: + # Update tags or other attributes if the node already exists + existing_node.tags.update(node.tags) + else: self.nodes_dict[node.id] = node self.nodes_list.append(node) return self.nodes_dict[node.id] diff --git a/package/tests/conftest.py b/package/tests/conftest.py index 5c1a300abb..ea25e94f7c 100644 --- a/package/tests/conftest.py +++ b/package/tests/conftest.py @@ -222,6 +222,7 @@ def example_pipeline_with_node_namespaces(): inputs=["raw_transaction_data", "cleaned_transaction_data"], outputs="validated_transaction_data", name="validation_node", + tags=["validation"], ), node( func=lambda validated_data, enrichment_data: ( @@ -381,6 +382,23 @@ def edge_case_example_pipelines( } +@pytest.fixture +def example_pipelines_with_additional_tags(example_pipeline_with_node_namespaces): + """ + Fixture to mock the use cases mentioned in + https://github.com/kedro-org/kedro-viz/issues/2106 + """ + + pipelines_dict = { + "pipeline": example_pipeline_with_node_namespaces, + "pipeline_with_tags": pipeline( + example_pipeline_with_node_namespaces, tags=["tag1", "tag2"] + ), + } + + yield pipelines_dict + + @pytest.fixture def expected_modular_pipeline_tree_for_edge_cases(): expected_tree_for_edge_cases_file_path = ( @@ -554,6 +572,41 @@ def example_api_for_edge_case_pipelines( yield api +@pytest.fixture +def example_api_for_pipelines_with_additional_tags( + data_access_manager: DataAccessManager, + example_pipelines_with_additional_tags: Dict[str, Pipeline], + example_catalog: DataCatalog, + session_store: BaseSessionStore, + mocker, +): + api = apps.create_api_app_from_project(mock.MagicMock()) + + # For readability we are not hashing the node id + mocker.patch("kedro_viz.utils._hash", side_effect=lambda value: value) + mocker.patch( + "kedro_viz.data_access.repositories.modular_pipelines._hash", + side_effect=lambda value: value, + ) + + populate_data( + data_access_manager, + example_catalog, + example_pipelines_with_additional_tags, + session_store, + {}, + ) + mocker.patch( + "kedro_viz.api.rest.responses.pipelines.data_access_manager", + new=data_access_manager, + ) + mocker.patch( + "kedro_viz.api.rest.responses.nodes.data_access_manager", + new=data_access_manager, + ) + yield api + + @pytest.fixture def example_transcoded_api( data_access_manager: DataAccessManager, diff --git a/package/tests/test_api/test_rest/test_responses/test_pipelines.py b/package/tests/test_api/test_rest/test_responses/test_pipelines.py index 4b933e33e2..b1d14d8ca3 100755 --- a/package/tests/test_api/test_rest/test_responses/test_pipelines.py +++ b/package/tests/test_api/test_rest/test_responses/test_pipelines.py @@ -35,6 +35,20 @@ def test_endpoint_main_no_default_pipeline(self, example_api_no_default_pipeline {"id": "data_processing", "name": "data_processing"}, ] + def test_endpoint_main_for_pipelines_with_additional_tags( + self, + example_api_for_pipelines_with_additional_tags, + ): + expected_tags = [ + {"id": "tag1", "name": "tag1"}, + {"id": "tag2", "name": "tag2"}, + {"id": "validation", "name": "validation"}, + ] + client = TestClient(example_api_for_pipelines_with_additional_tags) + response = client.get("/api/main") + actual_tags = response.json()["tags"] + assert actual_tags == expected_tags + def test_endpoint_main_for_edge_case_pipelines( self, example_api_for_edge_case_pipelines, From 4ec409efe9ab8f4fa72bcd6fa9150c46c3114810 Mon Sep 17 00:00:00 2001 From: Jitendra Gundaniya <38945204+jitu5@users.noreply.github.com> Date: Mon, 4 Nov 2024 11:07:36 +0000 Subject: [PATCH 3/5] Introduce `behaviour` prop object with `reFocus` prop (#2161) * Introduce modeOptions prop object with reFocus prop Signed-off-by: Jitendra Gundaniya * test fix Signed-off-by: Jitendra Gundaniya * Release note added Signed-off-by: Jitendra Gundaniya * Documentation Signed-off-by: Jitendra Gundaniya * Code review change Signed-off-by: Jitendra Gundaniya * Prop renamed to behaviour from modeOptions Signed-off-by: Jitendra Gundaniya * Revert container changes Signed-off-by: Jitendra Gundaniya --------- Signed-off-by: Jitendra Gundaniya --- README.npm.md | 6 +++++ RELEASE.md | 1 + src/components/app/app.js | 6 +++++ src/components/flowchart/flowchart.js | 29 ++++++++++++++-------- src/components/flowchart/flowchart.test.js | 1 + src/reducers/index.js | 1 + src/store/initial-state.js | 3 +++ 7 files changed, 36 insertions(+), 11 deletions(-) diff --git a/README.npm.md b/README.npm.md index f4632b88de..6ef817bc4b 100644 --- a/README.npm.md +++ b/README.npm.md @@ -140,6 +140,9 @@ The example below demonstrates how to configure your kedro-viz using different ` tag: { enabled: {companies: true} }, + behaviour: { + reFocus: true, + }, theme: "dark" }} /> @@ -161,6 +164,9 @@ The example below demonstrates how to configure your kedro-viz using different ` | `sidebar` | boolean | true | Show/Hide Sidebar and action toolbar | | `zoomToolbar` | boolean | true | Show/Hide zoom-in, zoom-out and zoom reset buttons together | | options.expandAllPipelines | boolean | false | Expand/Collapse Modular pipelines on first load | +| options.behaviour | | | | +| `reFocus` | boolean | true | In the flowchart, enable or disable the node re-focus behavior when clicking on nodes. + | options.nodeType | `{disabled: {parameters: boolean,task: boolean,data: boolean}}` | `{disabled: {parameters: true,task: false,data: false}}` | Configuration for node type options | | options.tag | `{enabled: {: boolean}}` | - | Configuration for tag options | | options.theme | string | dark | select `Kedro-Viz` theme : dark/light | diff --git a/RELEASE.md b/RELEASE.md index aaed5e9a5f..bf9e4cff8b 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -11,6 +11,7 @@ Please follow the established format: ## Major features and improvements - Update Kedro-Viz telemetry for opt-out model (#2022) +- Introduce `behaviour` prop object with `reFocus` prop (#2161) ## Bug fixes and other changes diff --git a/src/components/app/app.js b/src/components/app/app.js index 340dc1e44e..b1469854b6 100644 --- a/src/components/app/app.js +++ b/src/components/app/app.js @@ -119,6 +119,12 @@ App.propTypes = { tag: PropTypes.shape({ enabled: PropTypes.objectOf(PropTypes.bool), }), + /** + * Whether to re-focus the graph when a node is clicked + */ + behaviour: PropTypes.shape({ + reFocus: PropTypes.bool, + }), /** * Override the default enabled/disabled node types */ diff --git a/src/components/flowchart/flowchart.js b/src/components/flowchart/flowchart.js index 1afc5c4b93..58d300c30d 100644 --- a/src/components/flowchart/flowchart.js +++ b/src/components/flowchart/flowchart.js @@ -220,22 +220,28 @@ export class FlowChart extends Component { if (changed('edges', 'nodes', 'layers', 'chartSize', 'clickedNode')) { // Don't zoom out when the metadata or code panels are opened or closed - if (prevProps.visibleMetaSidebar !== this.props.visibleMetaSidebar) { + const metaSidebarViewChanged = + prevProps.visibleMetaSidebar !== this.props.visibleMetaSidebar; + + const codeViewChangedWithoutMetaSidebar = + prevProps.visibleCode !== this.props.visibleCode && + !this.props.visibleMetaSidebar; + + // Don't zoom out when the clicked node changes and the nodeReFocus is disabled + const clickedNodeChangedWithoutReFocus = + prevProps.clickedNode !== this.props.clickedNode && + !this.props.nodeReFocus; + + if ( + metaSidebarViewChanged || + codeViewChangedWithoutMetaSidebar || + clickedNodeChangedWithoutReFocus + ) { drawNodes.call(this, changed); drawEdges.call(this, changed); - return; } - if (prevProps.visibleCode !== this.props.visibleCode) { - if (!this.props.visibleMetaSidebar) { - drawNodes.call(this, changed); - drawEdges.call(this, changed); - - return; - } - } - this.resetView(preventZoom); } else { this.onChartZoomChanged(chartZoom); @@ -1000,6 +1006,7 @@ export const mapStateToProps = (state, ownProps) => ({ slicedPipeline: getSlicedPipeline(state), isSlicingPipelineApplied: state.slice.apply, visibleSlicing: state.visible.slicing, + nodeReFocus: state.behaviour.reFocus, runCommand: getRunCommand(state), ...ownProps, }); diff --git a/src/components/flowchart/flowchart.test.js b/src/components/flowchart/flowchart.test.js index fa9812df13..d3d8719fd3 100644 --- a/src/components/flowchart/flowchart.test.js +++ b/src/components/flowchart/flowchart.test.js @@ -492,6 +492,7 @@ describe('FlowChart', () => { runCommand: expect.any(Object), modularPipelineIds: expect.any(Object), visibleSlicing: expect.any(Boolean), + nodeReFocus: expect.any(Boolean), }; expect(mapStateToProps(mockState.spaceflights)).toEqual(expectedResult); }); diff --git a/src/reducers/index.js b/src/reducers/index.js index d2608fa252..79af193f24 100644 --- a/src/reducers/index.js +++ b/src/reducers/index.js @@ -89,6 +89,7 @@ const combinedReducer = combineReducers({ // These props don't have any actions associated with them display: createReducer(null), dataSource: createReducer(null), + behaviour: createReducer({}), edge: createReducer({}), // These props have very simple non-nested actions chartSize: createReducer({}, UPDATE_CHART_SIZE, 'chartSize'), diff --git a/src/store/initial-state.js b/src/store/initial-state.js index 70e1915b17..60f2423310 100644 --- a/src/store/initial-state.js +++ b/src/store/initial-state.js @@ -58,6 +58,9 @@ export const createInitialState = () => ({ zoomToolbar: true, metadataPanel: true, }, + behaviour: { + reFocus: true, + }, zoom: {}, runsMetadata: {}, }); From 96cf45abf63cc040e0bbf7e5a3193eeec375dea6 Mon Sep 17 00:00:00 2001 From: Jitendra Gundaniya <38945204+jitu5@users.noreply.github.com> Date: Mon, 4 Nov 2024 11:52:08 +0000 Subject: [PATCH 4/5] Replace `watchgod` library with `watchfiles` (#2134) * Package update from watchgod to watchfiles Signed-off-by: Jitendra Gundaniya * lint fix Signed-off-by: Jitendra Gundaniya * Test fix Signed-off-by: Jitendra Gundaniya * Test fixes Signed-off-by: Jitendra Gundaniya * lint fixes Signed-off-by: Jitendra Gundaniya * test fix Signed-off-by: Jitendra Gundaniya * test added Signed-off-by: Jitendra Gundaniya * lint fix Signed-off-by: Jitendra Gundaniya * Release note added Signed-off-by: Jitendra Gundaniya * AutoreloadFileFilter added for accurate auto reload files Signed-off-by: Jitendra Gundaniya * Test fix Signed-off-by: Jitendra Gundaniya * Test fix Signed-off-by: Jitendra Gundaniya * lint and unit test fix Signed-off-by: Jitendra Gundaniya * Test fix Signed-off-by: Jitendra Gundaniya * Lint fix Signed-off-by: Jitendra Gundaniya * Lint fix Signed-off-by: Jitendra Gundaniya * Fix import issue Signed-off-by: Jitendra Gundaniya * Extra import removed Signed-off-by: Jitendra Gundaniya * tmp_path fix for tests Signed-off-by: Jitendra Gundaniya * Lint fix Signed-off-by: Jitendra Gundaniya * Tests with docstring Signed-off-by: Jitendra Gundaniya * Moved to GitIgnoreSpec class Signed-off-by: Jitendra Gundaniya --------- Signed-off-by: Jitendra Gundaniya --- RELEASE.md | 1 + package/features/steps/lower_requirements.txt | 3 +- package/kedro_viz/autoreload_file_filter.py | 88 +++++++++++ package/kedro_viz/launchers/cli/run.py | 19 ++- package/kedro_viz/launchers/jupyter.py | 13 +- package/kedro_viz/server.py | 17 ++- package/requirements.txt | 3 +- package/test_requirements.txt | 1 + package/tests/test_autoreload_file_filter.py | 141 ++++++++++++++++++ .../tests/test_launchers/test_cli/test_run.py | 36 +++-- package/tests/test_launchers/test_jupyter.py | 1 + 11 files changed, 291 insertions(+), 32 deletions(-) create mode 100644 package/kedro_viz/autoreload_file_filter.py create mode 100644 package/tests/test_autoreload_file_filter.py diff --git a/RELEASE.md b/RELEASE.md index bf9e4cff8b..3261588f52 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -17,6 +17,7 @@ Please follow the established format: - Improve `kedro viz build` usage documentation (#2126) - Fix unserializable parameters value (#2122) +- Replace `watchgod` library with `watchfiles` and improve autoreload file watching filter (#2134) - Display full dataset type with library prefix in metadata panel (#2136) - Enable SQLite WAL mode for Azure ML to fix database locking issues (#2131) - Replace `flake8`, `isort`, `pylint` and `black` by `ruff` (#2149) diff --git a/package/features/steps/lower_requirements.txt b/package/features/steps/lower_requirements.txt index 1a149e7ad9..ab38585acf 100644 --- a/package/features/steps/lower_requirements.txt +++ b/package/features/steps/lower_requirements.txt @@ -3,7 +3,7 @@ fastapi==0.100.0 fsspec==2021.4 aiofiles==22.1.0 uvicorn[standard]==0.22.0 -watchgod==0.8.2 +watchfiles==0.24.0 plotly==4.8 packaging==23.0 pandas==1.3; python_version < '3.10' @@ -16,3 +16,4 @@ secure==0.3.0 # numpy 2.0 breaks with old versions of pandas and this # could be removed when the lowest version supported is updated numpy==1.26.4 +pathspec==0.12.1 diff --git a/package/kedro_viz/autoreload_file_filter.py b/package/kedro_viz/autoreload_file_filter.py new file mode 100644 index 0000000000..f8b13c6237 --- /dev/null +++ b/package/kedro_viz/autoreload_file_filter.py @@ -0,0 +1,88 @@ +""" +This module provides a custom file filter for autoreloading that filters out files based on allowed +file extensions and patterns specified in a .gitignore file. +""" + +import logging +from pathlib import Path +from typing import Optional, Set + +from pathspec import GitIgnoreSpec +from watchfiles import Change, DefaultFilter + +logger = logging.getLogger(__name__) + + +class AutoreloadFileFilter(DefaultFilter): + """ + Custom file filter for autoreloading that extends DefaultFilter. + Filters out files based on allowed file extensions and patterns specified in a .gitignore file. + """ + + allowed_extensions: Set[str] = {".py", ".yml", ".yaml", ".json"} + + def __init__(self, base_path: Optional[Path] = None): + """ + Initialize the AutoreloadFileFilter. + + Args: + base_path (Optional[Path]): The base path to set as the current working directory + for the filter. + """ + self.cwd = base_path or Path.cwd() + + # Call the superclass constructor + super().__init__() + + # Load .gitignore patterns + gitignore_path = self.cwd / ".gitignore" + try: + with open(gitignore_path, "r", encoding="utf-8") as gitignore_file: + ignore_patterns = gitignore_file.read().splitlines() + self.gitignore_spec: Optional[GitIgnoreSpec] = GitIgnoreSpec.from_lines( + "gitwildmatch", ignore_patterns + ) + except FileNotFoundError: + self.gitignore_spec = None + + def __call__(self, change: Change, path: str) -> bool: + """ + Determine whether a file change should be processed. + + Args: + change (Change): The type of change detected. + path (str): The path to the file that changed. + + Returns: + bool: True if the file should be processed, False otherwise. + """ + if not super().__call__(change, path): + logger.debug("Filtered out by DefaultFilter: %s", path) + return False + + path_obj = Path(path) + + # Exclude files matching .gitignore patterns + try: + relative_path = path_obj.resolve().relative_to(self.cwd.resolve()) + except ValueError: + logger.debug("Path not relative to CWD: %s", path) + return False + + try: + if self.gitignore_spec and self.gitignore_spec.match_file( + str(relative_path) + ): + logger.debug("Filtered out by .gitignore: %s", relative_path) + return False + # ruff: noqa: BLE001 + except Exception as exc: + logger.debug("Exception during .gitignore matching: %s", exc) + return True # Pass the file if .gitignore matching fails + + # Include only files with allowed extensions + if path_obj.suffix in self.allowed_extensions: + logger.debug("Allowed file: %s", path) + return True + logger.debug("Filtered out by allowed_extensions: %s", path_obj.suffix) + return False diff --git a/package/kedro_viz/launchers/cli/run.py b/package/kedro_viz/launchers/cli/run.py index e7dd08b408..b2e74a48be 100644 --- a/package/kedro_viz/launchers/cli/run.py +++ b/package/kedro_viz/launchers/cli/run.py @@ -7,6 +7,7 @@ from kedro.framework.cli.project import PARAMS_ARG_HELP from kedro.framework.cli.utils import _split_params +from kedro_viz.autoreload_file_filter import AutoreloadFileFilter from kedro_viz.constants import DEFAULT_HOST, DEFAULT_PORT from kedro_viz.launchers.cli.main import viz @@ -162,21 +163,25 @@ def run( "extra_params": params, "is_lite": lite, } + + process_context = multiprocessing.get_context("spawn") if autoreload: - from watchgod import RegExpWatcher, run_process + from watchfiles import run_process + run_process_args = [str(kedro_project_path)] run_process_kwargs = { - "path": kedro_project_path, "target": run_server, "kwargs": run_server_kwargs, - "watcher_cls": RegExpWatcher, - "watcher_kwargs": {"re_files": r"^.*(\.yml|\.yaml|\.py|\.json)$"}, + "watch_filter": AutoreloadFileFilter(), } - viz_process = multiprocessing.Process( - target=run_process, daemon=False, kwargs={**run_process_kwargs} + viz_process = process_context.Process( + target=run_process, + daemon=False, + args=run_process_args, + kwargs={**run_process_kwargs}, ) else: - viz_process = multiprocessing.Process( + viz_process = process_context.Process( target=run_server, daemon=False, kwargs={**run_server_kwargs} ) diff --git a/package/kedro_viz/launchers/jupyter.py b/package/kedro_viz/launchers/jupyter.py index 22af9fb99a..cd39610ab3 100644 --- a/package/kedro_viz/launchers/jupyter.py +++ b/package/kedro_viz/launchers/jupyter.py @@ -14,8 +14,9 @@ import IPython from IPython.display import HTML, display from kedro.framework.project import PACKAGE_NAME -from watchgod import RegExpWatcher, run_process +from watchfiles import run_process +from kedro_viz.autoreload_file_filter import AutoreloadFileFilter from kedro_viz.launchers.utils import _check_viz_up, _wait_for from kedro_viz.server import DEFAULT_HOST, DEFAULT_PORT, run_server @@ -146,15 +147,17 @@ def run_viz(args: str = "", local_ns: Dict[str, Any] = None) -> None: } process_context = multiprocessing.get_context("spawn") if autoreload: + run_process_args = [str(project_path)] run_process_kwargs = { - "path": project_path, "target": run_server, "kwargs": run_server_kwargs, - "watcher_cls": RegExpWatcher, - "watcher_kwargs": {"re_files": r"^.*(\.yml|\.yaml|\.py|\.json)$"}, + "watch_filter": AutoreloadFileFilter(), } viz_process = process_context.Process( - target=run_process, daemon=False, kwargs={**run_process_kwargs} + target=run_process, + daemon=False, + args=run_process_args, + kwargs={**run_process_kwargs}, ) else: viz_process = process_context.Process( diff --git a/package/kedro_viz/server.py b/package/kedro_viz/server.py index 251bb32b6b..db95289b6d 100644 --- a/package/kedro_viz/server.py +++ b/package/kedro_viz/server.py @@ -8,6 +8,7 @@ from kedro.io import DataCatalog from kedro.pipeline import Pipeline +from kedro_viz.autoreload_file_filter import AutoreloadFileFilter from kedro_viz.constants import DEFAULT_HOST, DEFAULT_PORT from kedro_viz.data_access import DataAccessManager, data_access_manager from kedro_viz.database import make_db_session_factory @@ -143,7 +144,7 @@ def run_server( import argparse import multiprocessing - from watchgod import RegExpWatcher, run_process + from watchfiles import run_process parser = argparse.ArgumentParser(description="Launch a development viz server") parser.add_argument("project_path", help="Path to a Kedro project") @@ -157,20 +158,24 @@ def run_server( project_path = (Path.cwd() / args.project_path).absolute() + run_process_args = [str(project_path)] run_process_kwargs = { - "path": project_path, "target": run_server, "kwargs": { "host": args.host, "port": args.port, "project_path": str(project_path), }, - "watcher_cls": RegExpWatcher, - "watcher_kwargs": {"re_files": r"^.*(\.yml|\.yaml|\.py|\.json)$"}, + "watch_filter": AutoreloadFileFilter(), } - viz_process = multiprocessing.Process( - target=run_process, daemon=False, kwargs={**run_process_kwargs} + process_context = multiprocessing.get_context("spawn") + + viz_process = process_context.Process( + target=run_process, + daemon=False, + args=run_process_args, + kwargs={**run_process_kwargs}, ) display_cli_message("Starting Kedro Viz ...", "green") diff --git a/package/requirements.txt b/package/requirements.txt index caf3fa63ea..16c7890f5d 100644 --- a/package/requirements.txt +++ b/package/requirements.txt @@ -15,4 +15,5 @@ secure>=0.3.0 sqlalchemy>=1.4, <3 strawberry-graphql>=0.192.0, <1.0 uvicorn[standard]>=0.30.0, <1.0 -watchgod>=0.8.2, <1.0 +watchfiles>=0.24.0 +pathspec>=0.12.1 \ No newline at end of file diff --git a/package/test_requirements.txt b/package/test_requirements.txt index 3260a24806..c2ac8e7c78 100644 --- a/package/test_requirements.txt +++ b/package/test_requirements.txt @@ -19,6 +19,7 @@ sqlalchemy-stubs~=0.4 strawberry-graphql[cli]>=0.99.0, <1.0 trufflehog~=2.2 httpx~=0.27.0 +pathspec>=0.12.1 # mypy types-aiofiles==0.1.3 diff --git a/package/tests/test_autoreload_file_filter.py b/package/tests/test_autoreload_file_filter.py new file mode 100644 index 0000000000..d5c9fb2ff7 --- /dev/null +++ b/package/tests/test_autoreload_file_filter.py @@ -0,0 +1,141 @@ +import logging +import shutil +import tempfile +from pathlib import Path +from unittest.mock import patch + +import pytest +from watchfiles import Change, DefaultFilter + +from kedro_viz.autoreload_file_filter import AutoreloadFileFilter + +logger = logging.getLogger(__name__) + + +@pytest.fixture +def file_filter(tmp_path): + """ + Fixture to create a temporary .gitignore file and initialize the AutoreloadFileFilter + with the test directory as the base path. + """ + # Create a .gitignore file + gitignore_path = tmp_path / ".gitignore" + gitignore_path.write_text("ignored.py\n") + + # Initialize the filter with the test directory as base_path + return AutoreloadFileFilter(base_path=tmp_path) + + +def test_no_gitignore(tmp_path): + """ + Test that a file passes the filter when the .gitignore file is missing. + """ + gitignored_file = tmp_path / "ignored.py" + gitignored_file.touch() + + # Initialize the filter without a .gitignore file + gitignore_path = tmp_path / ".gitignore" + if gitignore_path.exists(): + gitignore_path.unlink() + file_filter = AutoreloadFileFilter(base_path=tmp_path) + + result = file_filter(Change.modified, str(gitignored_file)) + assert result, "File should pass the filter when .gitignore is missing" + + +def test_gitignore_exception(file_filter, tmp_path): + """ + Test that a file passes the filter if an exception occurs during .gitignore matching. + """ + allowed_file = tmp_path / "test.py" + allowed_file.touch() + + with patch( + "pathspec.PathSpec.match_file", side_effect=Exception("Mocked exception") + ): + result = file_filter(Change.modified, str(allowed_file)) + assert result, "Filter should pass the file if .gitignore matching fails" + + +def test_allowed_file(file_filter, tmp_path): + """ + Test that a file with an allowed extension passes the filter. + """ + allowed_file = tmp_path / "test.py" + allowed_file.touch() + + result = file_filter(Change.modified, str(allowed_file)) + assert result, "Allowed file should pass the filter" + + +def test_disallowed_file(file_filter, tmp_path): + """ + Test that a file with a disallowed extension does not pass the filter. + """ + disallowed_file = tmp_path / "test.txt" + disallowed_file.touch() + + result = file_filter(Change.modified, str(disallowed_file)) + assert not result, "Disallowed file should not pass the filter" + + +def test_gitignored_file(file_filter, tmp_path): + """ + Test that a file listed in the .gitignore file does not pass the filter. + """ + gitignored_file = tmp_path / "ignored.py" + gitignored_file.touch() + + result = file_filter(Change.modified, str(gitignored_file)) + assert not result, "Gitignored file should not pass the filter" + + +def test_non_relative_path(file_filter): + """ + Test that a file outside the current working directory does not pass the filter. + """ + original_cwd = Path.cwd().parent # Go up one directory + outside_file = original_cwd / "outside.py" + outside_file.touch() + + result = file_filter(Change.modified, str(outside_file)) + assert not result, "File outside the CWD should not pass the filter" + + # Cleanup + outside_file.unlink() + + +def test_no_allowed_extension(file_filter, tmp_path): + """ + Test that a file without an allowed extension does not pass the filter. + """ + no_extension_file = tmp_path / "no_extension" + no_extension_file.touch() + + result = file_filter(Change.modified, str(no_extension_file)) + assert not result, "File without allowed extension should not pass the filter" + + +def test_directory_path(file_filter, tmp_path): + """ + Test that a directory does not pass the filter. + """ + directory_path = tmp_path / "some_directory" + directory_path.mkdir() + + result = file_filter(Change.modified, str(directory_path)) + assert not result, "Directories should not pass the filter" + + +def test_filtered_out_by_default_filter(file_filter, tmp_path, mocker): + """ + Test that a file is filtered out by the DefaultFilter. + """ + filtered_file = tmp_path / "filtered.py" + filtered_file.touch() + + # Mock the super().__call__ method to return False + mocker.patch.object(DefaultFilter, "__call__", return_value=False) + + result = file_filter(Change.modified, str(filtered_file)) + assert not result, "File should be filtered out by DefaultFilter" diff --git a/package/tests/test_launchers/test_cli/test_run.py b/package/tests/test_launchers/test_cli/test_run.py index b2d5c59b39..86adae92f6 100644 --- a/package/tests/test_launchers/test_cli/test_run.py +++ b/package/tests/test_launchers/test_cli/test_run.py @@ -4,9 +4,10 @@ import requests from click.testing import CliRunner from packaging.version import parse -from watchgod import RegExpWatcher, run_process +from watchfiles import run_process from kedro_viz import __version__ +from kedro_viz.autoreload_file_filter import AutoreloadFileFilter from kedro_viz.launchers.cli import main from kedro_viz.launchers.cli.run import _VIZ_PROCESSES from kedro_viz.launchers.utils import _PYPROJECT @@ -205,7 +206,10 @@ def test_kedro_viz_command_run_server( patched_check_viz_up, patched_start_browser, ): - process_init = mocker.patch("multiprocessing.Process") + mock_process_context = mocker.patch("multiprocessing.get_context") + mock_context_instance = mocker.Mock() + mock_process_context.return_value = mock_context_instance + mock_process = mocker.patch.object(mock_context_instance, "Process") runner = CliRunner() # Reduce the timeout argument from 600 to 1 to make test run faster. @@ -222,7 +226,7 @@ def test_kedro_viz_command_run_server( with runner.isolated_filesystem(): runner.invoke(main.viz_cli, command_options) - process_init.assert_called_once_with( + mock_process.assert_called_once_with( target=run_server, daemon=False, kwargs={**run_server_args} ) @@ -340,9 +344,15 @@ def test_kedro_viz_command_should_not_log_if_pypi_is_down( mock_click_echo.assert_has_calls(mock_click_echo_calls) def test_kedro_viz_command_with_autoreload( - self, mocker, mock_project_path, patched_check_viz_up, patched_start_browser + self, mocker, tmp_path, patched_check_viz_up, patched_start_browser ): - process_init = mocker.patch("multiprocessing.Process") + mock_process_context = mocker.patch("multiprocessing.get_context") + mock_context_instance = mocker.Mock() + mock_process_context.return_value = mock_context_instance + mock_process = mocker.patch.object(mock_context_instance, "Process") + mock_tmp_path = tmp_path / "tmp" + mock_tmp_path.mkdir() + mock_path = mock_tmp_path / "project_path" # Reduce the timeout argument from 600 to 1 to make test run faster. mocker.patch( @@ -351,14 +361,14 @@ def test_kedro_viz_command_with_autoreload( # Mock finding kedro project mocker.patch( "kedro_viz.launchers.utils._find_kedro_project", - return_value=mock_project_path, + return_value=mock_path, ) runner = CliRunner() with runner.isolated_filesystem(): runner.invoke(main.viz_cli, ["viz", "run", "--autoreload"]) + run_process_args = [str(mock_path)] run_process_kwargs = { - "path": mock_project_path, "target": run_server, "kwargs": { "host": "127.0.0.1", @@ -367,18 +377,20 @@ def test_kedro_viz_command_with_autoreload( "save_file": None, "pipeline_name": None, "env": None, + "project_path": mock_path, "autoreload": True, - "project_path": mock_project_path, "include_hooks": False, "package_name": None, "extra_params": {}, "is_lite": False, }, - "watcher_cls": RegExpWatcher, - "watcher_kwargs": {"re_files": "^.*(\\.yml|\\.yaml|\\.py|\\.json)$"}, + "watch_filter": mocker.ANY, } - process_init.assert_called_once_with( - target=run_process, daemon=False, kwargs={**run_process_kwargs} + mock_process.assert_called_once_with( + target=run_process, + daemon=False, + args=run_process_args, + kwargs={**run_process_kwargs}, ) assert run_process_kwargs["kwargs"]["port"] in _VIZ_PROCESSES diff --git a/package/tests/test_launchers/test_jupyter.py b/package/tests/test_launchers/test_jupyter.py index dd489778ca..485e7ff890 100644 --- a/package/tests/test_launchers/test_jupyter.py +++ b/package/tests/test_launchers/test_jupyter.py @@ -140,6 +140,7 @@ def test_run_viz_with_autoreload(self, mocker, patched_check_viz_up): mock_process.assert_called_once_with( target=mocker.ANY, daemon=False, # No daemon for autoreload + args=mocker.ANY, kwargs=mocker.ANY, ) From 02331cc96e7e199e12a33f07da92e82b149d4120 Mon Sep 17 00:00:00 2001 From: rashidakanchwala <37628668+rashidakanchwala@users.noreply.github.com> Date: Tue, 5 Nov 2024 09:25:47 +0000 Subject: [PATCH 5/5] Docs Fix on `--save-file` functionality (#2173) Resolves #1681 --- docs/source/kedro-viz_visualisation.md | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/docs/source/kedro-viz_visualisation.md b/docs/source/kedro-viz_visualisation.md index db244697ea..04556d7e4f 100644 --- a/docs/source/kedro-viz_visualisation.md +++ b/docs/source/kedro-viz_visualisation.md @@ -196,21 +196,25 @@ The visualisation now includes the layers: ## Share a pipeline visualisation -You can share a pipeline structure within a Kedro-Viz visualisation as a JSON file from the terminal: +You can save a pipeline structure within a Kedro-Viz visualisation directly from the terminal as follows: ```bash kedro viz run --save-file=my_shareable_pipeline ``` -This command will save a visualisation of the `__default__` pipeline as a JSON file called `my_shareable_pipeline.json`. It doesn't share data, such as that in the code panel, nor can you share images or charts. +This command saves your visualisation in a `my_shareable_pipeline` folder, which contains all pipeline and node information from your Kedro project. -To visualise the shared file, type the following to load it from the terminal: +To visualise your saved Kedro-Viz, load the `my_shareable_pipeline` folder from the terminal with: ```bash kedro viz run --load-file=my_shareable_pipeline ``` -You can also share a complete project visualisation, described in more detail on [the following page](./share_kedro_viz). +```{note} +This way of sharing requires a Kedro environment setup. + +For users who prefer not to set up a Kedro environment, [Kedro-Viz visualisations can also be shared via multiple hosting solutions](./share_kedro_viz). +``` ## Running Kedro-viz in a notebook.