Merge pull request #47 from Ouranosinc/fix-for-e2e-tests

Fix for e2e tests

CHANGES.rst

@@ -7,7 +7,12 @@ Changes
 `Unreleased <https://github.com/Ouranosinc/cowbird/tree/master>`_ (latest)
 ------------------------------------------------------------------------------------
 
-* Nothing yet.
+Bug Fixes
+~~~~~~~~~~~~~~~~~~~~~
+* The ``FileSystem`` handler now creates the WPS outputs data folder if it doesn't exist so that monitoring is setup
+  properly.
+* User permissions are set explicitly after creating his datastore folder to make sure the user can create and modify
+  files in it.
 
 `2.1.0 <https://github.com/Ouranosinc/cowbird/tree/2.1.0>`_ (2023-09-18)
 ------------------------------------------------------------------------------------

cowbird/handlers/impl/filesystem.py

@@ -78,11 +78,11 @@ def __init__(self,
             rf"^{self.wps_outputs_dir}/(?P<bird_name>\w+)/users/(?P<user_id>\d+)/(?P<subpath>.+)")
 
     def start_wps_outputs_monitoring(self, monitoring: Monitoring) -> None:
-        if os.path.exists(self.wps_outputs_dir):
-            LOGGER.info("Start monitoring WPS outputs folder [%s]", self.wps_outputs_dir)
-            monitoring.register(self.wps_outputs_dir, True, self)
-        else:
-            LOGGER.warning("Input WPS outputs folder [%s] does not exist.", self.wps_outputs_dir)
+        if not os.path.exists(self.wps_outputs_dir):
+            LOGGER.warning("Input WPS outputs folder [%s] does not exist. Creating folder...", self.wps_outputs_dir)
+            os.makedirs(self.wps_outputs_dir)
+        LOGGER.info("Start monitoring WPS outputs folder [%s]", self.wps_outputs_dir)
+        monitoring.register(self.wps_outputs_dir, True, self)
 
     def get_user_workspace_dir(self, user_name: str) -> str:
         return os.path.join(self.workspace_dir, user_name)
@@ -261,8 +261,8 @@ def _create_wps_outputs_hardlink(self, src_path: str, overwrite: bool = False,
                                                    subpath=regex_match.group("subpath"))
             api_services = magpie_handler.get_services_by_type(ServiceAPI.service_type)
             if self.secure_data_proxy_name not in api_services:
-                LOGGER.warning("`%s` service not found. Considering user WPS outputs data as accessible by default.",
-                               self.secure_data_proxy_name)
+                LOGGER.warning("`%s` service not found. Considering user WPS outputs data as accessible (read-only) "
+                               "by default.", self.secure_data_proxy_name)
                 apply_new_path_permissions(src_path, True, False, False)
             else:  # get access and apply permissions if the secure-data-proxy exists
                 access_allowed = self.update_secure_data_proxy_path_perms(src_path, user_name)

cowbird/handlers/impl/geoserver.py

@@ -736,6 +736,11 @@ def _create_datastore_dir(self, workspace_name: str) -> None:
             os.mkdir(datastore_folder_path)
         except FileExistsError:
             LOGGER.info("User datastore directory already existing (skip creation): [%s]", datastore_folder_path)
+        # Make sure user can read and write files in his datastore folder
+        apply_new_path_permissions(datastore_folder_path,
+                                   is_readable=True,
+                                   is_writable=True,
+                                   is_executable=True)
 
     @geoserver_response_handling
     def _create_datastore_request(self,

cowbird/monitoring/monitoring.py

@@ -123,3 +123,10 @@ def unregister(self, path: str, cb_monitor: Union[FSMonitor, Type[FSMonitor], st
             except KeyError:
                 pass
         return False
+
+    def unregister_all(self) -> None:
+        """
+        Stops and unregisters all monitors.
+        """
+        self.monitors.clear()
+        self.store.clear_services(drop=False)

docs/components.rst

@@ -106,8 +106,8 @@ If the file does not have any read or write permissions, the hardlink will not b
     Refer to `DAC-571 <https://crim-ca.atlassian.net/browse/DAC-571>`_ for more details on the design choices for the
     management of permissions.
 
-If no ``secure-data-proxy`` service is found, the user files are assumed to be fully available
-with read and write permissions for the user.
+If no ``secure-data-proxy`` service is found, all user files are assumed to be available with read permissions for
+the user.
 
 Note that different design choices were made to respect the constraints of the file system and to prevent the user from
 accessing forbidden data:

setup.cfg

@@ -59,11 +59,12 @@ exclude =
 	eggs,
 	parts,
 	share,
+	node_modules,
 
 [pylint]
 
 [bandit]
-exclude = *.egg-info,build,dist,env,./tests,test_*
+exclude = *.egg-info,build,dist,env,./tests,test_*,./node_modules
 targets = .
 
 [tool:isort]

tests/test_filesystem.py

@@ -190,7 +190,7 @@ def test_create_user_missing_workspace_dir(self, mock_head_request):
                     "active": True,
                     "workspace_dir": workspace_dir,
                     "jupyterhub_user_data_dir": self.jupyterhub_user_data_dir,
-                    "wps_outputs_dir": "/wps_outputs"}}})
+                    "wps_outputs_dir": self.wps_outputs_dir}}})
         data = {
             "event": "created",
             "user_name": self.test_username,

tests/test_monitoring.py

@@ -35,8 +35,8 @@ def setUpClass(cls):
         with cls.cfg_file as f:
             f.write(yaml.safe_dump({"handlers": {"": {}}}))  # no specific handler required for these tests
         cls.app = get_test_app(settings={"cowbird.config_path": cls.cfg_file.name})
-        # clear up monitor entries from db
-        Monitoring().store.clear_services(drop=False)
+        # clear up monitor entries
+        Monitoring().unregister_all()
 
     @classmethod
     def tearDownClass(cls):