feat: store writer framework IDE into project directory instead of ui…

….json

* chore: move wf_project methods into its own module

src/writer/app_runner.py

@@ -18,16 +18,8 @@
 from pydantic import ValidationError
 from watchdog.observers.polling import PollingObserver
 
-from writer import VERSION, audit_and_fix
-from writer.core import (
-    EventHandlerRegistry,
-    MiddlewareRegistry,
-    WriterSession,
-    use_request_context,
-    wf_project_migrate_obsolete_ui_json,
-    wf_project_read_files,
-    wf_project_write_files,
-)
+from writer import VERSION, audit_and_fix, wf_project
+from writer.core import EventHandlerRegistry, MiddlewareRegistry, WriterSession, use_request_context
 from writer.core_ui import ingest_bmc_component_tree
 from writer.ss_types import (
     AppProcessServerRequest,
@@ -690,13 +682,13 @@ def _load_persisted_script(self) -> str:
 
     def _load_persisted_components(self) -> Dict[str, ComponentDefinition]:
         if os.path.isfile(os.path.join(self.app_path, "ui.json")):
-            wf_project_migrate_obsolete_ui_json(self.app_path)
+            wf_project.migrate_obsolete_ui_json(self.app_path)
 
         if not os.path.isdir(os.path.join(self.app_path, ".wf")):
             logging.error("Couldn't find .wf in the path provided: %s.", self.app_path)
             sys.exit(1)
 
-        _, components = wf_project_read_files(self.app_path)
+        _, components = wf_project.read_files(self.app_path)
         components = audit_and_fix.fix_components(components)
         return components
 
@@ -720,7 +712,7 @@ async def update_components(self, session_id: str, payload: ComponentUpdateReque
                 "Cannot update components in non-update mode.")
         self.bmc_components = payload.components
 
-        wf_project_write_files(self.app_path, metadata={"writer_version": VERSION}, components=payload.components)
+        wf_project.write_files(self.app_path, metadata={"writer_version": VERSION}, components=payload.components)
 
         return await self.dispatch_message(session_id, ComponentUpdateRequest(
             type="componentUpdate",

src/writer/command_line.py

@@ -7,11 +7,6 @@
 import click
 
 import writer.serve
-from writer.core import (
-    wf_project_migrate_obsolete_ui_json,
-    wf_project_read_files,
-    wf_project_write_files,
-)
 from writer.deploy import cloud
 
 CONTEXT_SETTINGS = {'help_option_names': ['-h', '--help']}

src/writer/core.py

@@ -11,12 +11,10 @@
 import logging
 import math
 import multiprocessing
-import os
 import re
 import secrets
 import time
 import traceback
-import types
 import urllib.request
 from abc import ABCMeta
 from contextvars import ContextVar
@@ -47,13 +45,11 @@
 from writer import core_ui
 from writer.core_ui import Component
 from writer.ss_types import (
-    ComponentDefinition,
     DataframeRecordAdded,
     DataframeRecordRemoved,
     DataframeRecordUpdated,
     InstancePath,
     InstancePathItem,
-    MetadataDefinition,
     Readable,
     WriterEvent,
     WriterEventResult,
@@ -2423,107 +2419,6 @@ def writer_event_handler_build_arguments(func: Callable, writer_args: dict) -> L
 
     return func_args
 
-def wf_project_write_files(app_path: str, metadata: MetadataDefinition, components: dict[str, ComponentDefinition]) -> None:
-    """
-    Writes the meta data of the WF project to the `.wf` directory (metadata, components, ...).
-
-    * the metadata.json file is written in json format
-    * a file for the root component written in jsonline format
-    * one file per page is created in the form `components-{id}.json` in jsonline format
-
-    >>> wf_project_write_files('app/hello', metadata={"writer_version": "0.1" }, components=...)
-    """
-    wf_directory = os.path.join(app_path, ".wf")
-    if not os.path.exists(wf_directory):
-        os.makedirs(wf_directory)
-
-    with io.open(os.path.join(wf_directory, "metadata.json"), "w") as f:
-        json.dump(metadata, f, indent=4)
-
-    root_component = components["root"]
-    with io.open(os.path.join(wf_directory, "components-root.jsonl"), "w") as f:
-        f.write(json.dumps(root_component))
-
-    list_pages = []
-    for c in components.values():
-        if c["type"] == "page":
-            list_pages.append(c["id"])
-
-    for position, page_id in enumerate(list_pages):
-        page_components = []
-        page_components_ids = {page_id}
-        for c in components.values():
-            if _lookup_page_for_component(components, c['id']) in page_components_ids:
-                page_components.append(c)
-                page_components_ids.add(c["id"])
-
-        with io.open(os.path.join(wf_directory, f"components-page-{position}-{page_id}.jsonl"), "w") as f:
-            for p in page_components:
-                f.write(json.dumps(p) + "\n")
-
-def wf_project_read_files(app_path: str) -> Tuple[MetadataDefinition, dict[str, ComponentDefinition]]:
-    """
-    Reads project files in the `.wf` folder.
-
-    The components are read in page order.
-
-    >>> metadata, components = wf_project_read_files('app/hello')
-    """
-    components: dict[str, ComponentDefinition] = {}
-
-    meta_data_path = os.path.join(app_path, ".wf", "metadata.json")
-    try:
-        with io.open(meta_data_path, "r") as filep:
-            metadata: MetadataDefinition = json.load(filep)
-    except Exception as e:
-        raise ValueError(f"Error reading metadata file {meta_data_path} : {e}")
-
-    root_component_path = os.path.join(app_path, ".wf", "components-root.jsonl")
-    try:
-        with io.open(root_component_path, "r") as filep:
-            root_component: ComponentDefinition = json.loads(filep.read())
-            components.update({root_component["id"]: root_component})
-    except Exception as e:
-        raise ValueError(f"Error reading root component file {root_component_path} : {e}")
-
-    files = os.listdir(os.path.join(app_path, ".wf"))
-    page_files = [file for file in files if file.startswith("components-page-")]
-    sorted_page_files = sorted(page_files, key=lambda x: int(x.split("-")[2]))
-    for page_file in sorted_page_files:
-        page_file_path = os.path.join(app_path, ".wf", page_file)
-        try:
-            with io.open(page_file_path, "r") as filep:
-                for line in filep:
-                    component: ComponentDefinition = json.loads(line)
-                    components.update({component["id"]: component})
-        except Exception as e:
-            raise ValueError(f"Error reading page component file {page_file_path} : {e}")
-
-    return metadata, components
-
-
-def wf_project_migrate_obsolete_ui_json(app_path: str) -> None:
-    """
-    Migrates a project that uses ui.json file to the current project format
-
-    The ui.json file is removed after the migration.
-    """
-    assert os.path.isfile(os.path.join(app_path, "ui.json")), f"ui.json file required for migration into {app_path}"
-
-    logger = logging.getLogger('writer')
-    with io.open(os.path.join(app_path, "ui.json"), "r") as f:
-        parsed_file = json.load(f)
-
-    if not isinstance(parsed_file, dict):
-        raise ValueError("No dictionary found in components file.")
-
-    file_payload = parsed_file
-    metadata = file_payload.get("metadata", {})
-    components = file_payload.get("components", {})
-    wf_project_write_files(app_path, metadata, components)
-    os.remove(os.path.join(app_path, "ui.json"))
-    logger.warning('project format has changed and has been migrated with success. ui.json file has been removed.')
-
 
 def writer_event_handler_invoke(callable_handler: Callable, writer_args: dict) -> Any:
     """
@@ -2637,29 +2532,6 @@ def _event_handler_ui_manager():
     return WriterUIManager()
 
 
-def _lookup_page_for_component(components: dict[str, ComponentDefinition], component_id: str) -> Optional[str]:
-    """
-    Retrieves the page of a component
-
-    >>> _lookup_page_for_component(components, "6a490318-239e-4fe9-a56b-f0f33d628c87")
-    """
-    component = components[component_id]
-    parent_id = component.get("parentId")
-    if parent_id is None:
-        return None
-
-    if component['type'] == "page":
-        return component['id']
-
-    if parent_id in components:
-        if components[parent_id]['type'] == "page":
-            return parent_id
-        else:
-            return _lookup_page_for_component(components, parent_id)
-
-    return None
-
-
 def _split_record_as_pandas_record_and_index(param: dict, index_columns: list) -> Tuple[dict, tuple]:
     """
     Separates a record into the record part and the index part to be able to

src/writer/core_ui.py

@@ -9,6 +9,8 @@
 from pydantic import BaseModel, Field
 from typing_extensions import TypedDict
 
+from writer.ss_types import ComponentDefinition
+
 current_parent_container: ContextVar[Union["Component", None]] = \
     ContextVar("current_parent_container")
 
@@ -384,6 +386,32 @@ def ingest_bmc_component_tree(component_tree: ComponentTree, components: Dict[st
     component_tree.ingest(components, tree=Branch.bmc)
 
 
+def lookup_page_for_component(components: Dict[str, ComponentDefinition], component_id: str) -> Optional[str]:
+    """
+    Retrieves the page of a component
+
+    >>> lookup_page_for_component(components, "6a490318-239e-4fe9-a56b-f0f33d628c87")
+    """
+    component: Optional[ComponentDefinition] = components.get(component_id, None)
+    if component is None:
+        return None
+
+    parent_id = component.get("parentId")
+    if parent_id is None:
+        return None
+
+    if component['type'] == "page":
+        return component['id']
+
+    if parent_id in components:
+        if components[parent_id]['type'] == "page":
+            return parent_id
+        else:
+            return lookup_page_for_component(components, parent_id)
+
+    return None
+
+
 def cmc_components_list(component_tree: ComponentTree) -> list:
     """
     Returns the list of code managed components in the component tree.

src/writer/wf_project.py

@@ -0,0 +1,120 @@
+"""
+This module manipulates the folder of a wf project stored into `wf`.
+
+>>> wf_project.write_files('app/hello', metadata={"writer_version": "0.1" }, components=...)
+
+>>> metadata, components = wf_project.read_files('app/hello')
+"""
+import io
+import json
+import logging
+import os
+from typing import Tuple
+
+from writer import core_ui
+from writer.ss_types import ComponentDefinition, MetadataDefinition
+
+
+def write_files(app_path: str, metadata: MetadataDefinition, components: dict[str, ComponentDefinition]) -> None:
+    """
+    Writes the meta data of the WF project to the `.wf` directory (metadata, components, ...).
+
+    * the metadata.json file is written in json format
+    * a file for the root component written in jsonline format
+    * one file per page is created in the form `components-{id}.json` in jsonline format
+
+    >>> wf_project.write_files('app/hello', metadata={"writer_version": "0.1" }, components=...)
+    """
+    wf_directory = os.path.join(app_path, ".wf")
+    if not os.path.exists(wf_directory):
+        os.makedirs(wf_directory)
+
+    with io.open(os.path.join(wf_directory, "metadata.json"), "w") as f:
+        json.dump(metadata, f, indent=4)
+
+    root_component = components["root"]
+    with io.open(os.path.join(wf_directory, "components-root.jsonl"), "w") as f:
+        f.write(json.dumps(root_component))
+
+    list_pages = []
+    for c in components.values():
+        if c["type"] == "page":
+            list_pages.append(c["id"])
+
+    for position, page_id in enumerate(list_pages):
+        page_components = []
+        page_components_ids = {page_id}
+        for c in components.values():
+            if core_ui.lookup_page_for_component(components, c['id']) in page_components_ids:
+                page_components.append(c)
+                page_components_ids.add(c["id"])
+
+        with io.open(os.path.join(wf_directory, f"components-page-{position}-{page_id}.jsonl"), "w") as f:
+            for p in page_components:
+                f.write(json.dumps(p) + "\n")
+
+def read_files(app_path: str) -> Tuple[MetadataDefinition, dict[str, ComponentDefinition]]:
+    """
+    Reads project files in the `.wf` folder.
+
+    The components are read in page order.
+
+    >>> metadata, components = wf_project.read_files('app/hello')
+    """
+    components: dict[str, ComponentDefinition] = {}
+
+    meta_data_path = os.path.join(app_path, ".wf", "metadata.json")
+    try:
+        with io.open(meta_data_path, "r") as filep:
+            metadata: MetadataDefinition = json.load(filep)
+    except Exception as e:
+        raise ValueError(f"Error reading metadata file {meta_data_path} : {e}")
+
+    root_component_path = os.path.join(app_path, ".wf", "components-root.jsonl")
+    try:
+        with io.open(root_component_path, "r") as filep:
+            root_component: ComponentDefinition = json.loads(filep.read())
+            components.update({root_component["id"]: root_component})
+    except Exception as e:
+        raise ValueError(f"Error reading root component file {root_component_path} : {e}")
+
+    files = os.listdir(os.path.join(app_path, ".wf"))
+    page_files = [file for file in files if file.startswith("components-page-")]
+    sorted_page_files = sorted(page_files, key=lambda x: int(x.split("-")[2]))
+    for page_file in sorted_page_files:
+        page_file_path = os.path.join(app_path, ".wf", page_file)
+        try:
+            with io.open(page_file_path, "r") as filep:
+                for line in filep:
+                    component: ComponentDefinition = json.loads(line)
+                    components.update({component["id"]: component})
+        except Exception as e:
+            raise ValueError(f"Error reading page component file {page_file_path} : {e}")
+
+    return metadata, components
+
+
+def migrate_obsolete_ui_json(app_path: str) -> None:
+    """
+    Migrates a project that uses ui.json file to the current project format
+
+    The ui.json file is removed after the migration.
+
+    >>> wf_project.migrate_obsolete_ui_json('app/hello')
+    """
+    assert os.path.isfile(os.path.join(app_path, "ui.json")), f"ui.json file required for migration into {app_path}"
+
+    logger = logging.getLogger('writer')
+    with io.open(os.path.join(app_path, "ui.json"), "r") as f:
+        parsed_file = json.load(f)
+
+    if not isinstance(parsed_file, dict):
+        raise ValueError("No dictionary found in components file.")
+
+    file_payload = parsed_file
+    metadata = file_payload.get("metadata", {})
+    components = file_payload.get("components", {})
+    write_files(app_path, metadata, components)
+    os.remove(os.path.join(app_path, "ui.json"))
+    logger.warning('project format has changed and has been migrated with success. ui.json file has been removed.')
+