Allow users to provide copying of optional files without failing. (#41)

Co-authored-by: DavidNew-NOAA <[email protected]>

.vscode/launch.json

@@ -0,0 +1,15 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Python Debugger: Current File",
+            "type": "debugpy",
+            "request": "launch",
+            "program": "${file}",
+            "console": "integratedTerminal"
+        }
+    ]
+}

.vscode/settings.json

@@ -0,0 +1,11 @@
+{
+    "python.testing.unittestEnabled": false,
+    "python.testing.pytestEnabled": true,
+    "python.testing.autoTestDiscoverOnSaveEnabled": true,
+    "python.testing.promptToConfigure": true,
+    "python.testing.pytestArgs": ["-v", "--color=yes", "--no-cov"],
+    "python.testing.cwd": "./tests",
+    "pre-commit-helper.config": "${workspaceFolder}/.pre-commit-config.yaml",
+    "pre-commit-helper.runOnSave": "all hooks",
+    "pre-commit-helper.runOnSaveRegex": "^(.*)\\.(py|c|h)$"
+}

setup.cfg

@@ -67,5 +67,5 @@ run-coverage = true
 addopts = --cov=wxflow --cov-report=term-missing --cov-report=xml --cov-report=html
 
 [tool:pycodestyle]
-exclude = .git,.github,venv,.vscode,docs/conf.py
+exclude = .git,.github,.venv,venv,.vscode,docs/conf.py
 max-line-length = 160

src/wxflow/file_utils.py

@@ -1,3 +1,4 @@
+import os
 from logging import getLogger
 
 from .fsutils import cp, mkdir
@@ -17,13 +18,20 @@ class FileHandler:
 
     NOTE
     ----
-    "action" can be one of mkdir", "copy", etc.
+    "action" can be one of "mkdir", "copy", "copy_req", "copy_opt", etc.
     Corresponding "act" would be ['dir1', 'dir2'], [['src1', 'dest1'], ['src2', 'dest2']]
+    "copy_req" will raise an error if the source file does not exist
+    "copy_opt" will not raise an error if the source file does not exist but will present a warning
 
     Attributes
     ----------
     config : dict
             Dictionary of files to manipulate
+
+    NOTE
+    ----
+    `copy` will be deprecated in the future in favor of `copy_req` and `copy_opt`
+    Users are encouraged to use `copy_req` and `copy_opt` instead of `copy`
     """
 
     def __init__(self, config):
@@ -35,15 +43,25 @@ def sync(self):
         Method to execute bulk actions on files described in the configuration
         """
         sync_factory = {
-            'copy': self._copy_files,
+            'copy': self.copy_req,
+            'copy_req': self.copy_req,
+            'copy_opt': self.copy_opt,
             'mkdir': self._make_dirs,
         }
         # loop through the configuration keys
         for action, files in self.config.items():
             sync_factory[action](files)
 
     @staticmethod
-    def _copy_files(filelist):
+    def copy_req(filelist):
+        FileHandler._copy_files(filelist, required=True)
+
+    @staticmethod
+    def copy_opt(filelist):
+        FileHandler._copy_files(filelist, required=False)
+
+    @staticmethod
+    def _copy_files(filelist, required=True):
         """Function to copy all files specified in the list
 
         `filelist` should be in the form:
@@ -53,15 +71,28 @@ def _copy_files(filelist):
         ----------
         filelist : list
                 List of lists of [src, dest]
+        required : bool, optional
+                Flag to indicate if the src file is required to exist. Default is True
         """
         for sublist in filelist:
             if len(sublist) != 2:
-                raise Exception(
+                raise IndexError(
                     f"List must be of the form ['src', 'dest'], not {sublist}")
             src = sublist[0]
             dest = sublist[1]
-            cp(src, dest)
-            logger.info(f'Copied {src} to {dest}')
+            if os.path.exists(src):
+                try:
+                    cp(src, dest)
+                    logger.info(f'Copied {src} to {dest}')
+                except Exception as ee:
+                    logger.exception(f"Error copying {src} to {dest}")
+                    raise ee(f"Error copying {src} to {dest}")
+            else:
+                if required:
+                    logger.exception(f"Source file '{src}' does not exist and is required, ABORT!")
+                    raise FileNotFoundError(f"Source file '{src}' does not exist")
+                else:
+                    logger.warning(f"Source file '{src}' does not exist, skipping!")
 
     @staticmethod
     def _make_dirs(dirlist):
@@ -73,5 +104,9 @@ def _make_dirs(dirlist):
                 List of directories to create
         """
         for dd in dirlist:
-            mkdir(dd)
-            logger.info(f'Created {dd}')
+            try:
+                mkdir(dd)
+                logger.info(f'Created {dd}')
+            except Exception as ee:
+                logger.exception(f"Error creating directory {dd}")
+                raise ee(f"Error creating directory {dd}")

tests/conftest.py

@@ -0,0 +1,4 @@
+import os
+import sys
+
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '../src')))

tests/test_file_utils.py

@@ -65,3 +65,18 @@ def test_copy(tmp_path):
     # Check if files were indeed copied
     for ff in dest_files:
         assert os.path.isfile(ff)
+
+    # Create a config object for copying optional non-existent files (c.txt does not exist)
+    copy_list.append([input_dir_path / 'c.txt', output_dir_path / 'c.txt'])
+    config = {'copy_opt': copy_list}
+
+    # Copy input files to output files
+    FileHandler(config).sync()
+
+    # Create a config object for copying required non-existent files (c.txt does not exist)
+    config = {'copy_req': copy_list}
+    try:
+        FileHandler(config).sync()
+    except FileNotFoundError as e:
+        c_file = input_dir_path / 'c.txt'
+        assert f"Source file '{c_file}' does not exist" in str(e)