feat: store writer framework IDE into project directory instead of ui…

….json

* chore: move wf_project methods into its own module

alfred/apps.py

@@ -2,16 +2,13 @@
 
 import alfred
 
+from writer import wf_project
+
 
 @alfred.command('apps.update', help='update the apps in the repo to the latest version')
 @alfred.option('--app', help='the path of the app to update')
 def apps_update(app: str = None):
     import writer
-    from writer.core import (
-        wf_project_migrate_obsolete_ui_json,
-        wf_project_read_files,
-        wf_project_write_files,
-    )
 
     if app is not None:
         apps = [app]
@@ -36,12 +33,12 @@ def apps_update(app: str = None):
 
         if os.path.isfile(os.path.join(abs_path, "ui.json")):
             print(f'{app} : migrate ui.json')
-            wf_project_migrate_obsolete_ui_json(abs_path)
+            wf_project.migrate_obsolete_ui_json(abs_path)
 
-        metadata, components = wf_project_read_files(abs_path)
+        metadata, components = wf_project.read_files(abs_path)
         if metadata.get('writer_version') == writer.VERSION:
             print("The app is already up to date")
         else:
             metadata['writer_version'] = writer.VERSION
-            wf_project_write_files(abs_path, metadata, components)
+            wf_project.write_files(abs_path, metadata, components)
             print(f"{app} : app is up to date")

src/writer/app_runner.py

@@ -18,16 +18,8 @@
 from pydantic import ValidationError
 from watchdog.observers.polling import PollingObserver
 
-from writer import VERSION, audit_and_fix
-from writer.core import (
-    EventHandlerRegistry,
-    MiddlewareRegistry,
-    WriterSession,
-    use_request_context,
-    wf_project_migrate_obsolete_ui_json,
-    wf_project_read_files,
-    wf_project_write_files,
-)
+from writer import VERSION, audit_and_fix, wf_project
+from writer.core import EventHandlerRegistry, MiddlewareRegistry, WriterSession, use_request_context
 from writer.core_ui import ingest_bmc_component_tree
 from writer.ss_types import (
     AppProcessServerRequest,
@@ -678,25 +670,27 @@ async def dispatch_message(self, session_id: Optional[str], request: AppProcessS
         return response
 
     def _load_persisted_script(self) -> str:
+        logger = logging.getLogger('writer')
         try:
             contents = None
             with open(os.path.join(self.app_path, "main.py"), "r", encoding='utf-8') as f:
                 contents = f.read()
             return contents
         except FileNotFoundError:
-            logging.error(
+            logger.error(
                 "Couldn't find main.py in the path provided: %s.", self.app_path)
             sys.exit(1)
 
     def _load_persisted_components(self) -> Dict[str, ComponentDefinition]:
+        logger = logging.getLogger('writer')
         if os.path.isfile(os.path.join(self.app_path, "ui.json")):
-            wf_project_migrate_obsolete_ui_json(self.app_path)
+            wf_project.migrate_obsolete_ui_json(self.app_path)
 
         if not os.path.isdir(os.path.join(self.app_path, ".wf")):
-            logging.error("Couldn't find .wf in the path provided: %s.", self.app_path)
+            logger.error("Couldn't find .wf in the path provided: %s.", self.app_path)
             sys.exit(1)
 
-        _, components = wf_project_read_files(self.app_path)
+        _, components = wf_project.read_files(self.app_path)
         components = audit_and_fix.fix_components(components)
         return components
 
@@ -720,7 +714,7 @@ async def update_components(self, session_id: str, payload: ComponentUpdateReque
                 "Cannot update components in non-update mode.")
         self.bmc_components = payload.components
 
-        wf_project_write_files(self.app_path, metadata={"writer_version": VERSION}, components=payload.components)
+        wf_project.write_files(self.app_path, metadata={"writer_version": VERSION}, components=payload.components)
 
         return await self.dispatch_message(session_id, ComponentUpdateRequest(
             type="componentUpdate",

src/writer/command_line.py

@@ -7,11 +7,6 @@
 import click
 
 import writer.serve
-from writer.core import (
-    wf_project_migrate_obsolete_ui_json,
-    wf_project_read_files,
-    wf_project_write_files,
-)
 from writer.deploy import cloud
 
 CONTEXT_SETTINGS = {'help_option_names': ['-h', '--help']}

src/writer/core.py

@@ -11,12 +11,10 @@
 import logging
 import math
 import multiprocessing
-import os
 import re
 import secrets
 import time
 import traceback
-import types
 import urllib.request
 from abc import ABCMeta
 from contextvars import ContextVar
@@ -47,13 +45,11 @@
 from writer import core_ui
 from writer.core_ui import Component
 from writer.ss_types import (
-    ComponentDefinition,
     DataframeRecordAdded,
     DataframeRecordRemoved,
     DataframeRecordUpdated,
     InstancePath,
     InstancePathItem,
-    MetadataDefinition,
     Readable,
     WriterEvent,
     WriterEventResult,
@@ -2423,107 +2419,6 @@ def writer_event_handler_build_arguments(func: Callable, writer_args: dict) -> L
 
     return func_args
 
-def wf_project_write_files(app_path: str, metadata: MetadataDefinition, components: dict[str, ComponentDefinition]) -> None:
-    """
-    Writes the meta data of the WF project to the `.wf` directory (metadata, components, ...).
-
-    * the metadata.json file is written in json format
-    * a file for the root component written in jsonline format
-    * one file per page is created in the form `components-{id}.json` in jsonline format
-
-    >>> wf_project_write_files('app/hello', metadata={"writer_version": "0.1" }, components=...)
-    """
-    wf_directory = os.path.join(app_path, ".wf")
-    if not os.path.exists(wf_directory):
-        os.makedirs(wf_directory)
-
-    with io.open(os.path.join(wf_directory, "metadata.json"), "w") as f:
-        json.dump(metadata, f, indent=4)
-
-    root_component = components["root"]
-    with io.open(os.path.join(wf_directory, "components-root.jsonl"), "w") as f:
-        f.write(json.dumps(root_component))
-
-    list_pages = []
-    for c in components.values():
-        if c["type"] == "page":
-            list_pages.append(c["id"])
-
-    for position, page_id in enumerate(list_pages):
-        page_components = []
-        page_components_ids = {page_id}
-        for c in components.values():
-            if _lookup_page_for_component(components, c['id']) in page_components_ids:
-                page_components.append(c)
-                page_components_ids.add(c["id"])
-
-        with io.open(os.path.join(wf_directory, f"components-page-{position}-{page_id}.jsonl"), "w") as f:
-            for p in page_components:
-                f.write(json.dumps(p) + "\n")
-
-def wf_project_read_files(app_path: str) -> Tuple[MetadataDefinition, dict[str, ComponentDefinition]]:
-    """
-    Reads project files in the `.wf` folder.
-
-    The components are read in page order.
-
-    >>> metadata, components = wf_project_read_files('app/hello')
-    """
-    components: dict[str, ComponentDefinition] = {}
-
-    meta_data_path = os.path.join(app_path, ".wf", "metadata.json")
-    try:
-        with io.open(meta_data_path, "r") as filep:
-            metadata: MetadataDefinition = json.load(filep)
-    except Exception as e:
-        raise ValueError(f"Error reading metadata file {meta_data_path} : {e}")
-
-    root_component_path = os.path.join(app_path, ".wf", "components-root.jsonl")
-    try:
-        with io.open(root_component_path, "r") as filep:
-            root_component: ComponentDefinition = json.loads(filep.read())
-            components.update({root_component["id"]: root_component})
-    except Exception as e:
-        raise ValueError(f"Error reading root component file {root_component_path} : {e}")
-
-    files = os.listdir(os.path.join(app_path, ".wf"))
-    page_files = [file for file in files if file.startswith("components-page-")]
-    sorted_page_files = sorted(page_files, key=lambda x: int(x.split("-")[2]))
-    for page_file in sorted_page_files:
-        page_file_path = os.path.join(app_path, ".wf", page_file)
-        try:
-            with io.open(page_file_path, "r") as filep:
-                for line in filep:
-                    component: ComponentDefinition = json.loads(line)
-                    components.update({component["id"]: component})
-        except Exception as e:
-            raise ValueError(f"Error reading page component file {page_file_path} : {e}")
-
-    return metadata, components
-
-
-def wf_project_migrate_obsolete_ui_json(app_path: str) -> None:
-    """
-    Migrates a project that uses ui.json file to the current project format
-
-    The ui.json file is removed after the migration.
-    """
-    assert os.path.isfile(os.path.join(app_path, "ui.json")), f"ui.json file required for migration into {app_path}"
-
-    logger = logging.getLogger('writer')
-    with io.open(os.path.join(app_path, "ui.json"), "r") as f:
-        parsed_file = json.load(f)
-
-    if not isinstance(parsed_file, dict):
-        raise ValueError("No dictionary found in components file.")
-
-    file_payload = parsed_file
-    metadata = file_payload.get("metadata", {})
-    components = file_payload.get("components", {})
-    wf_project_write_files(app_path, metadata, components)
-    os.remove(os.path.join(app_path, "ui.json"))
-    logger.warning('project format has changed and has been migrated with success. ui.json file has been removed.')
-
 
 def writer_event_handler_invoke(callable_handler: Callable, writer_args: dict) -> Any:
     """
@@ -2637,29 +2532,6 @@ def _event_handler_ui_manager():
     return WriterUIManager()
 
 
-def _lookup_page_for_component(components: dict[str, ComponentDefinition], component_id: str) -> Optional[str]:
-    """
-    Retrieves the page of a component
-
-    >>> _lookup_page_for_component(components, "6a490318-239e-4fe9-a56b-f0f33d628c87")
-    """
-    component = components[component_id]
-    parent_id = component.get("parentId")
-    if parent_id is None:
-        return None
-
-    if component['type'] == "page":
-        return component['id']
-
-    if parent_id in components:
-        if components[parent_id]['type'] == "page":
-            return parent_id
-        else:
-            return _lookup_page_for_component(components, parent_id)
-
-    return None
-
-
 def _split_record_as_pandas_record_and_index(param: dict, index_columns: list) -> Tuple[dict, tuple]:
     """
     Separates a record into the record part and the index part to be able to

src/writer/core_ui.py

@@ -9,6 +9,8 @@
 from pydantic import BaseModel, Field
 from typing_extensions import TypedDict
 
+from writer.ss_types import ComponentDefinition
+
 current_parent_container: ContextVar[Union["Component", None]] = \
     ContextVar("current_parent_container")
 
@@ -384,6 +386,32 @@ def ingest_bmc_component_tree(component_tree: ComponentTree, components: Dict[st
     component_tree.ingest(components, tree=Branch.bmc)
 
 
+def lookup_page_for_component(components: Dict[str, ComponentDefinition], component_id: str) -> Optional[str]:
+    """
+    Retrieves the page of a component
+
+    >>> lookup_page_for_component(components, "6a490318-239e-4fe9-a56b-f0f33d628c87")
+    """
+    component: Optional[ComponentDefinition] = components.get(component_id, None)
+    if component is None:
+        return None
+
+    parent_id = component.get("parentId")
+    if parent_id is None:
+        return None
+
+    if component['type'] == "page":
+        return component['id']
+
+    if parent_id in components:
+        if components[parent_id]['type'] == "page":
+            return parent_id
+        else:
+            return lookup_page_for_component(components, parent_id)
+
+    return None
+
+
 def cmc_components_list(component_tree: ComponentTree) -> list:
     """
     Returns the list of code managed components in the component tree.