Merge pull request #102 from jwlodek/add-create-dir-depth

Add two records: CreateDirectories and DirectoryExists. Automatically create directories based on this PV like in AD
PandABlocks · Jun 5, 2024 · 8e25d64 · 8e25d64
2 parents c08325a + 0289178
commit 8e25d64
Show file tree

Hide file tree

Showing 5 changed files with 281 additions and 10 deletions.
diff --git a/docs/how-to/capture-hdf.md b/docs/how-to/capture-hdf.md
@@ -9,6 +9,8 @@ These can be viewed from the DATA screen.
 ```
 
 - The file directory and name are chosen with `:DATA:HDFDirectory` and `:DATA:HDFFileName`.
+- The number of directories that the IOC is allowed to create provided they don't exist is determined by `:DATA:CreateDirectory`. The behavior of this signal is the same as the identical PV in [`areaDetector`](https://areadetector.github.io/areaDetector/ADCore/NDPluginFile.html).
+- `DATA:DirectoryExists` represents whether or not the directory specified exists and is writable by the user under which the IOC is running.
 - `:DATA:NumCapture` is the number of frames to capture in the file.
 - `:DATA:NumCaptured` is the number of frames written to file.
 - `:DATA:NumReceived` is the number of frames received from the panda.

diff --git a/src/pandablocks_ioc/_hdf_ioc.py b/src/pandablocks_ioc/_hdf_ioc.py
@@ -5,6 +5,7 @@
 from collections import deque
 from enum import Enum
 from importlib.util import find_spec
+from pathlib import Path
 from typing import Callable, Deque, Optional, Union
 
 from pandablocks.asyncio import AsyncioClient
@@ -311,6 +312,8 @@ class HDF5RecordController:
  _client: AsyncioClient
 
  _directory_record: RecordWrapper
+ _create_directory_record: RecordWrapper
+ _directory_exists_record: RecordWrapper
  _file_name_record: RecordWrapper
  _file_number_record: RecordWrapper
  _file_format_record: RecordWrapper
@@ -340,7 +343,8 @@ def __init__(self, client: AsyncioClient, record_prefix: str):
  length=path_length,
  DESC="File path for HDF5 files",
  validate=self._parameter_validate,
- on_update=self._update_full_file_path,
+ on_update=self._update_directory_path,
+ always_update=True,
  )
  add_automatic_pvi_info(
  PviGroup.HDF,
@@ -352,6 +356,40 @@ def __init__(self, client: AsyncioClient, record_prefix: str):
  record_prefix + ":" + self._DATA_PREFIX + ":HDFDirectory"
  )
 
+ create_directory_record_name = EpicsName(self._DATA_PREFIX + ":CreateDirectory")
+ self._create_directory_record = builder.longOut(
+ create_directory_record_name,
+ initial_value=0,
+ DESC="Directory creation depth",
+ )
+ add_automatic_pvi_info(
+ PviGroup.HDF,
+ self._create_directory_record,
+ create_directory_record_name,
+ builder.longOut,
+ )
+ self._create_directory_record.add_alias(
+ record_prefix + ":" + create_directory_record_name.upper()
+ )
+
+ directory_exists_name = EpicsName(self._DATA_PREFIX + ":DirectoryExists")
+ self._directory_exists_record = builder.boolIn(
+ directory_exists_name,
+ ZNAM="No",
+ ONAM="Yes",
+ initial_value=0,
+ DESC="Directory exists",
+ )
+ add_automatic_pvi_info(
+ PviGroup.HDF,
+ self._directory_exists_record,
+ directory_exists_name,
+ builder.boolIn,
+ )
+ self._directory_exists_record.add_alias(
+ record_prefix + ":" + directory_exists_name.upper()
+ )
+
  file_name_record_name = EpicsName(self._DATA_PREFIX + ":HDF_FILE_NAME")
  self._file_name_record = builder.longStringOut(
  file_name_record_name,
@@ -523,6 +561,70 @@ def _parameter_validate(self, record: RecordWrapper, new_val) -> bool:
  return False
  return True
 
+ async def _update_directory_path(self, new_val) -> None:
+ """Handles writes to the directory path PV, creating
+ directories based on the setting of the CreateDirectory record"""
+ new_path = Path(new_val).absolute()
+ create_dir_depth = self._create_directory_record.get()
+ max_dirs_to_create = 0
+ if create_dir_depth < 0:
+ max_dirs_to_create = abs(create_dir_depth)
+ elif create_dir_depth > len(new_path.parents):
+ max_dirs_to_create = 0
+ elif create_dir_depth > 0:
+ max_dirs_to_create = len(new_path.parents) - create_dir_depth
+
+ logging.debug(f"Permitted to create up to {max_dirs_to_create} dirs.")
+ dirs_to_create = 0
+ for p in reversed(new_path.parents):
+ if not p.exists():
+ if dirs_to_create == 0:
+ # First directory level that does not exist, log it.
+ logging.error(f"All dir from {str(p)} and below do not exist!")
+ dirs_to_create += 1
+ else:
+ logging.info(f"{str(p)} exists")
+
+ # Account for target path itself not existing
+ if not os.path.exists(new_path):
+ dirs_to_create += 1
+
+ logging.debug(f"Need to create {dirs_to_create} directories.")
+
+ # Case where all dirs exist
+ if dirs_to_create == 0:
+ if os.access(new_path, os.W_OK):
+ status_msg = "Dir exists and is writable"
+ self._directory_exists_record.set(1)
+ else:
+ status_msg = "Dirs exist but aren't writable."
+ self._directory_exists_record.set(0)
+ # Case where we will create directories
+ elif dirs_to_create <= max_dirs_to_create:
+ logging.debug(f"Attempting to create {dirs_to_create} dir(s)...")
+ try:
+ os.makedirs(new_path, exist_ok=True)
+ status_msg = f"Created {dirs_to_create} dirs."
+ self._directory_exists_record.set(1)
+ except PermissionError:
+ status_msg = "Permission error creating dirs!"
+ self._directory_exists_record.set(0)
+ # Case where too many directories need to be created
+ else:
+ status_msg = f"Need to create {dirs_to_create} > {max_dirs_to_create} dirs."
+ self._directory_exists_record.set(0)
+
+ if self._directory_exists_record.get() == 0:
+ sevr = alarm.MAJOR_ALARM, alrm = alarm.STATE_ALARM
+ logging.error(status_msg)
+ else:
+ sevr = alarm.NO_ALARM, alrm = alarm.NO_ALARM
+ logging.debug(status_msg)
+
+ self._status_message_record.set(status_msg, severity=sevr, alarm=alrm)
+
+ await self._update_full_file_path(new_val)
+
  async def _update_full_file_path(self, new_val) -> None:
  self._full_file_path_record.set(self._get_filepath())
 
@@ -532,6 +634,12 @@ async def _handle_hdf5_data(self) -> None:
  This method expects to be run as an asyncio Task."""
  try:
  # Set up the hdf buffer
+
+ if not self._directory_exists_record.get() == 1:
+ raise RuntimeError(
+ "Configured HDF directory does not exist or is not writable!"
+ )
+
  num_capture: int = self._num_capture_record.get()
  capture_mode: CaptureMode = CaptureMode(self._capture_mode_record.get())
  filepath = self._get_filepath()

diff --git a/tests/conftest.py b/tests/conftest.py
@@ -1,4 +1,23 @@
 # flake8: noqa
 
+import os
+
 from fixtures.mocked_panda import *
 from fixtures.panda_data import *
+
+
+# Autouse fixture that will set all EPICS networking env vars to use lo interface
+# to avoid false failures caused by things like firewalls blocking EPICS traffic.
+@pytest.fixture(scope="session", autouse=True)
+def configure_epics_environment():
+ os.environ["EPICS_CAS_INTF_ADDR_LIST"] = "127.0.0.1"
+ os.environ["EPICS_CAS_BEACON_ADDR_LIST"] = "127.0.0.1"
+ os.environ["EPICS_CA_ADDR_LIST"] = "127.0.0.1"
+ os.environ["EPICS_CAS_AUTO_ADDR_LIST"] = "NO"
+ os.environ["EPICS_CA_AUTO_BEACON_ADDR_LIST"] = "NO"
+
+ os.environ["EPICS_PVAS_INTF_ADDR_LIST"] = "127.0.0.1"
+ os.environ["EPICS_PVAS_BEACON_ADDR_LIST"] = "127.0.0.1"
+ os.environ["EPICS_PVA_ADDR_LIST"] = "127.0.0.1"
+ os.environ["EPICS_PVAS_AUTO_BEACON_ADDR_LIST"] = "NO"
+ os.environ["EPICS_PVA_AUTO_ADDR_LIST"] = "NO"
diff --git a/tests/test-bobfiles/DATA.bob b/tests/test-bobfiles/DATA.bob
@@ -3,7 +3,7 @@
  <x>0</x>
  <y>0</y>
  <width>506</width>
- <height>413</height>
+ <height>463</height>
  <grid_step_x>4</grid_step_x>
  <grid_step_y>4</grid_step_y>
  <widget type="label" version="2.0.0">
@@ -30,7 +30,7 @@
  <x>5</x>
  <y>30</y>
  <width>496</width>
- <height>106</height>
+ <height>156</height>
  <transparent>true</transparent>
  <widget type="label" version="2.0.0">
  <name>Label</name>
@@ -52,35 +52,73 @@
  </widget>
  <widget type="label" version="2.0.0">
  <name>Label</name>
- <text>Hdf File Name</text>
+ <text>Createdirectory</text>
  <x>0</x>
  <y>25</y>
  <width>250</width>
  <height>20</height>
  </widget>
  <widget type="textentry" version="3.0.0">
  <name>TextEntry</name>
- <pv_name>TEST_PREFIX:DATA:HDF_FILE_NAME</pv_name>
+ <pv_name>TEST_PREFIX:DATA:CreateDirectory</pv_name>
  <x>255</x>
  <y>25</y>
  <width>205</width>
  <height>20</height>
  <horizontal_alignment>1</horizontal_alignment>
+ </widget>
+ <widget type="label" version="2.0.0">
+ <name>Label</name>
+ <text>Directoryexists</text>
+ <x>0</x>
+ <y>50</y>
+ <width>250</width>
+ <height>20</height>
+ </widget>
+ <widget type="textupdate" version="2.0.0">
+ <name>TextUpdate</name>
+ <pv_name>TEST_PREFIX:DATA:DirectoryExists</pv_name>
+ <x>255</x>
+ <y>50</y>
+ <width>205</width>
+ <height>20</height>
+ <font>
+ <font name="Default Bold" family="Liberation Sans" style="BOLD" size="14.0">
+ </font>
+ </font>
+ <horizontal_alignment>1</horizontal_alignment>
+ </widget>
+ <widget type="label" version="2.0.0">
+ <name>Label</name>
+ <text>Hdf File Name</text>
+ <x>0</x>
+ <y>75</y>
+ <width>250</width>
+ <height>20</height>
+ </widget>
+ <widget type="textentry" version="3.0.0">
+ <name>TextEntry</name>
+ <pv_name>TEST_PREFIX:DATA:HDF_FILE_NAME</pv_name>
+ <x>255</x>
+ <y>75</y>
+ <width>205</width>
+ <height>20</height>
+ <horizontal_alignment>1</horizontal_alignment>
  <format>6</format>
  </widget>
  <widget type="label" version="2.0.0">
  <name>Label</name>
  <text>Hdf Full File Path</text>
  <x>0</x>
- <y>50</y>
+ <y>100</y>
  <width>250</width>
  <height>20</height>
  </widget>
  <widget type="textupdate" version="2.0.0">
  <name>TextUpdate</name>
  <pv_name>TEST_PREFIX:DATA:HDF_FULL_FILE_PATH</pv_name>
  <x>255</x>
- <y>50</y>
+ <y>100</y>
  <width>205</width>
  <height>20</height>
  <font>
@@ -94,7 +132,7 @@
  <widget type="group" version="2.0.0">
  <name>CAPTURE</name>
  <x>5</x>
- <y>141</y>
+ <y>191</y>
  <width>496</width>
  <height>206</height>
  <transparent>true</transparent>
@@ -268,7 +306,7 @@
  <widget type="group" version="2.0.0">
  <name>OUTPUTS</name>
  <x>5</x>
- <y>352</y>
+ <y>402</y>
  <width>496</width>
  <height>56</height>
  <transparent>true</transparent>