(NEX-11354) Move /etc/dispatcher_state to `/var/intel-manageability…

…/dispatcher_state` (with backwards compatibility) (#556)

inbm/Changelog.md

@@ -18,6 +18,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
  - Updated 'cryptography' Python library in dispatcher to 43.0.1, fixing GHSA-h4gh-qq45-vh27.
 
 ### Changed
+ - Removed all references to `future` library as we do not use Python 2
+ - (NEX-11354) Moved `/etc/dispatcher_state` to `/var/intel-manageability/dispatcher_state`
  - (NEXMANAGE-744) Remove `psutil` in favor if `shutil.disk_usage` to save space
  - (NEXMANAGE-744) Don't pull in `dmidecode` in `inbm-lib` -- pull in only in `telemetry`
  - (NEXMANAGE-744) Removed all references to `future` library as we do not use Python 2

inbm/dispatcher-agent/dispatcher/common/constants.py

@@ -4,6 +4,7 @@
  Copyright (C) 2017-2024 Intel Corporation
  SPDX-License-Identifier: Apache-2.0
 """
-from inbm_lib.path_prefixes import INTEL_MANAGEABILITY_RAW_ETC
+from inbm_lib.path_prefixes import INTEL_MANAGEABILITY_RAW_ETC, INTEL_MANAGEABILITY_VAR_PATH_PREFIX
 
-DISPATCHER_STATE_FILE = str(INTEL_MANAGEABILITY_RAW_ETC / 'dispatcher_state')
+NEW_DISPATCHER_STATE_FILE = str(INTEL_MANAGEABILITY_VAR_PATH_PREFIX / 'dispatcher_state')
+OLD_DISPATCHER_STATE_FILE = str(INTEL_MANAGEABILITY_RAW_ETC / 'dispatcher_state')

inbm/dispatcher-agent/dispatcher/common/dispatcher_state.py

@@ -16,18 +16,24 @@
 
 from typing import Dict, Any, TypedDict
 
-from .constants import DISPATCHER_STATE_FILE
+from .constants import OLD_DISPATCHER_STATE_FILE, NEW_DISPATCHER_STATE_FILE
 from ..dispatcher_exception import DispatcherException
 from inbm_common_lib.utility import remove_file
 
 logger = logging.getLogger(__name__)
 
 
 def clear_dispatcher_state() -> None:
- """Deletes the dispatcher state file which was created at start of a previous SOTA"""
- if os.path.exists(DISPATCHER_STATE_FILE):
- logger.debug('Dispatcher state file exists')
- remove_file(DISPATCHER_STATE_FILE)
+ """Deletes the dispatcher state file which was created at start of a previous SOTA.
+ 
+ Check both locations for the file and delete the first one found."""
+ if os.path.exists(OLD_DISPATCHER_STATE_FILE):
+ logger.debug('Dispatcher state file exists in old location')
+ remove_file(OLD_DISPATCHER_STATE_FILE)
+ logger.debug('Dispatcher state file deleted from disk')
+ elif os.path.exists(NEW_DISPATCHER_STATE_FILE):
+ logger.debug('Dispatcher state file exists in new location')
+ remove_file(NEW_DISPATCHER_STATE_FILE)
  logger.debug('Dispatcher state file deleted from disk')
  else:
  logger.debug('Dispatcher state file does not exist')
@@ -37,9 +43,11 @@ def is_dispatcher_state_file_exists() -> bool:
  """SOTA leaves a file on disk just after an attempt for SOTA. If file exists, it means
  reboot was triggered by SOTA attempt.
 
+ Check both old and new locations.
+
  @return: True if file exists; otherwise, false.
  """
- if os.path.exists(DISPATCHER_STATE_FILE):
+ if os.path.exists(OLD_DISPATCHER_STATE_FILE) or os.path.exists(NEW_DISPATCHER_STATE_FILE):
  logger.debug('Dispatcher state file exists')
  return True
  else:
@@ -59,49 +67,80 @@ def is_dispatcher_state_file_exists() -> bool:
 def consume_dispatcher_state_file(read: bool = False) -> DispatcherState | None:
  """Read dispatcher state file and return state object, clearing state file on success
 
- @param read: set to True when only file info needs to be read without removing the file
- @return: state object from state file; None on error
+ Try old location first, then new location.
+
+ @param read: Set to True to read file info without removing the file
+ @return: State object from state file; None on error
  """
- logger.debug("")
- fd = None
+ logger.debug("Starting consume_dispatcher_state_file")
+
  state = None
- try:
- logger.debug("attempting to open file " + str(DISPATCHER_STATE_FILE))
- fd = builtins.open(DISPATCHER_STATE_FILE, 'rb')
- logger.debug("attempting to unpickle from state file")
- # override S301 because we are unpickling from trusted file
- state = pickle.load(fd) # nosec
- logger.debug("unpickling succeeded")
- logger.debug(f"Disp State file info: {state}")
- except (OSError, pickle.UnpicklingError, AttributeError, FileNotFoundError) as e:
- logger.exception(f"Exception while extracting dispatcher state from disk: {e}")
- raise DispatcherException("Exception while extracting dispatcher state from disk") from e
- finally:
- if fd:
- fd.close()
- if read:
- return state
- clear_dispatcher_state()
+ state_files = [OLD_DISPATCHER_STATE_FILE, NEW_DISPATCHER_STATE_FILE]
+
+ for state_file in state_files:
+ try:
+ logger.debug(f"Attempting to open file {state_file}")
+ with builtins.open(state_file, 'rb') as fd:
+ logger.debug("Attempting to unpickle from state file")
+ state = pickle.load(fd) # nosec
+ logger.debug("Unpickling succeeded")
+ logger.debug(f"Dispatcher State file info: {state}")
+ # Successfully read the state file, no need to try others
+ break
+ except (OSError, pickle.UnpicklingError, AttributeError, FileNotFoundError) as e:
+ logger.exception(f"Exception while extracting dispatcher state from {state_file}: {e}")
+ state = None # Ensure state is None if this attempt fails
+
+ if state is None:
+ logger.error("Failed to extract dispatcher state from all state files.")
+ raise DispatcherException("Exception while extracting dispatcher state from disk")
+
+ if not read:
+ try:
+ logger.debug("Clearing dispatcher state files")
+ clear_dispatcher_state()
+ except Exception as e:
+ logger.exception(f"Failed to clear dispatcher state files: {e}")
+ # Depending on requirements, you might want to raise or handle this differently
+
  return state
 
 
 def write_dispatcher_state_to_state_file(state: DispatcherState) -> None: # pragma: no cover
  """Update state file dictionary with state object
 
+ If the old state file exists, read it in, update state, write to new location,
+ and delete the old state file.
+
  @param state: state object to use to update state file
  """
-
  try:
- # if there's already a state file, read it in so we can update it and write back
- if is_dispatcher_state_file_exists():
- with builtins.open(DISPATCHER_STATE_FILE, 'rb') as fd:
- # we are reading from a trusted state file here
- val = pickle.load(fd) # nosec B301
- val.update(state)
- state = val
- logger.debug(f"STATE WRITTEN: {state}")
- with open(DISPATCHER_STATE_FILE, 'wb') as fd:
+ # Initialize existing_state to empty dict
+ existing_state: DispatcherState = DispatcherState()
+ # Check for state file in both old and new locations
+ state_file_found = False
+ for state_file in [OLD_DISPATCHER_STATE_FILE, NEW_DISPATCHER_STATE_FILE]:
+ if os.path.exists(state_file):
+ with builtins.open(state_file, 'rb') as fd:
+ # Reading from a trusted state file here
+ existing_state = pickle.load(fd) # nosec B301
+ state_file_found = True
+ break # Stop after finding the first existing state file
+
+ if state_file_found:
+ existing_state.update(state)
+ state = existing_state
+
+ # Write the updated state to the new location
+ with open(NEW_DISPATCHER_STATE_FILE, 'wb') as fd:
  pickle.dump(state, fd)
+
+ # After writing, remove the old state file if it exists
+ if os.path.exists(OLD_DISPATCHER_STATE_FILE):
+ remove_file(OLD_DISPATCHER_STATE_FILE)
+ logger.debug('Removed old dispatcher state file')
+
  except (OSError, pickle.UnpicklingError, AttributeError) as e:
  logger.exception(f"Exception while saving dispatcher state to disk: {e}")
  raise DispatcherException("Exception while saving dispatcher state to disk") from e
+

inbm/dispatcher-agent/fpm-template/etc/apparmor.d/usr.bin.inbm-dispatcher

@@ -62,6 +62,7 @@
  /usr/bin/gpg rUx,
  /bin/gpg rUx,
  /etc/dispatcher_state rw,
+ /var/intel-manageability/dispatcher_state rw,
  /etc/lsb-release r,
  /etc/magic r,
  /etc/mime.types r,

inbm/integration-reloaded/test/fota/FOTA_GOOD_AMI_TEST_postboot.sh

@@ -23,8 +23,8 @@ echo "Checking health of all services"
 check_health_tc_services
 echo "Health check passed"
 
-echo Also looking for /etc/dispatcher_state to be gone
-! [ -f /etc/dispatcher_state ]
+echo Also looking for /var/intel-manageability/dispatcher_state to be gone
+! [ -f /var/intel-manageability/dispatcher_state ]
 
 echo "Cleaning up after test"
 cleanup_after_test

inbm/integration-reloaded/test/fota/FOTA_GOOD_INBC_PLATFORM_INFO_MATCH_postboot.sh

@@ -23,8 +23,8 @@ echo "Checking health of all services"
 check_health_tc_services
 echo "Health check passed"
 
-echo Also looking for /etc/dispatcher_state to be gone
-! [ -f /etc/dispatcher_state ]
+echo Also looking for /var/intel-manageability/dispatcher_state to be gone
+! [ -f /var/intel-manageability/dispatcher_state ]
 
 echo "Cleaning up after test"
 cleanup_after_test

inbm/integration-reloaded/test/fota/FOTA_GOOD_LOCAL_TEST_postboot.sh

@@ -27,8 +27,8 @@ echo After reboot, we are looking for exactly 1 capsule file in /boot/efi/
 ls /boot/efi/
 ls /boot/efi/ | wc -l | grep 1
 
-echo Also looking for /etc/dispatcher_state to be gone
-! [ -f /etc/dispatcher_state ]
+echo Also looking for /var/intel-manageability/dispatcher_state to be gone
+! [ -f /var/intel-manageability/dispatcher_state ]
 
 
 echo "Cleaning up after test"

inbm/integration-reloaded/test/fota/FOTA_GOOD_TEST_NO_TAR_postboot.sh

@@ -19,8 +19,8 @@ done
 
 sleep 3
 
-echo Also looking for /etc/dispatcher_state to be gone
-! [ -f /etc/dispatcher_state ]
+echo Also looking for /var/intel-manageability/dispatcher_state to be gone
+! [ -f /var/intel-manageability/dispatcher_state ]
 
 echo "Checking health of all services"
 check_health_tc_services

inbm/integration-reloaded/test/fota/FOTA_GOOD_TEST_postboot.sh

@@ -27,8 +27,8 @@ echo After reboot, we are looking for exactly 1 capsule file in /boot/efi/
 ls /boot/efi/
 ls /boot/efi/ | wc -l | grep 1
 
-echo Also looking for /etc/dispatcher_state to be gone
-! [ -f /etc/dispatcher_state ]
+echo Also looking for /var/intel-manageability/dispatcher_state to be gone
+! [ -f /var/intel-manageability/dispatcher_state ]
 
 
 echo "Cleaning up after test"

inbm/integration-reloaded/test/pota/POTA_TEST_SUCCESS_WITH_LOCAL_TAG.sh

@@ -30,7 +30,7 @@ cp ${VAGRANT_INPUT_PATH}/${BIOS_FILE} /var/cache/manageability/repository-tool
 
 mkdir -p /var/log/sota_test
 
-! [ -f /etc/dispatcher_state ]
+! [ -f /var/intel-manageability/dispatcher_state ]
 touch /etc/force_yocto
 ls -l /etc/force_yocto
 date

inbm/integration-reloaded/test/pota/POTA_TEST_SUCCESS_WITH_REMOTE_TAG.sh

@@ -27,7 +27,7 @@ cp /scripts/afulnx_64 /opt/afulnx/afulnx_64
 
 mkdir -p /var/log/sota_test
 
-! [ -f /etc/dispatcher_state ]
+! [ -f /var/intel-manageability/dispatcher_state ]
 touch /etc/force_yocto
 ls -l /etc/force_yocto
 date

inbm/integration-reloaded/test/reboot/REBOOT_GOOD_TEST_POSTBOOT.sh

@@ -27,7 +27,7 @@ echo "Waiting for dispatcher to come up postboot..." | systemd-cat
 sleep 3
 echo "Done waiting for dispatcher." | systemd-cat
 
-echo Also looking for /etc/dispatcher_state to be gone
-! [ -f /etc/dispatcher_state ]
+echo Also looking for /var/intel-manageability/dispatcher_state to be gone
+! [ -f /var/intel-manageability/dispatcher_state ]
 
 cleanup_after_test

inbm/integration-reloaded/test/sota/SOTA_NO_DOWNLOAD_SUCCESS_postboot.sh

@@ -27,11 +27,11 @@ echo "Waiting for dispatcher to finish with sota update success postboot..." | s
 sleep 30
 echo "Done waiting for dispatcher." | systemd-cat
 
-echo looking for /etc/dispatcher_state to be gone
-if [ -f /etc/dispatcher_state ] ; then
- echo ERROR: /etc/dispatcher_state should be removed and is not | systemd-cat
- ls -l /etc/dispatcher_state
- cat /etc/dispatcher_state
+echo looking for /var/intel-manageability/dispatcher_state to be gone
+if [ -f /var/intel-manageability/dispatcher_state ] ; then
+ echo ERROR: /var/intel-manageability/dispatcher_state should be removed and is not | systemd-cat
+ ls -l /var/intel-manageability/dispatcher_state
+ cat /var/intel-manageability/dispatcher_state
  echo "<FAILED> SOTA NO_DOWNLOAD SUCCESS TEST"
  exit 1
 fi

inbm/integration-reloaded/test/sota/SOTA_UPDATE_SUCCESS_NO_SNAPSHOT_postboot.sh

@@ -32,11 +32,11 @@ echo After reboot, we are looking for exactly 5 files in /var/lib/dispatcher/upl
 ls /var/lib/dispatcher/upload/
 ls /var/lib/dispatcher/upload/ | wc -l | grep 5
 
-echo Also looking for /etc/dispatcher_state to be gone
-if [ -f /etc/dispatcher_state ] ; then
- echo ERROR: /etc/dispatcher_state should be removed and is not | systemd-cat
- ls -l /etc/dispatcher_state
- cat /etc/dispatcher_state
+echo Also looking for /var/intel-manageability/dispatcher_state to be gone
+if [ -f /var/intel-manageability/dispatcher_state ] ; then
+ echo ERROR: /var/intel-manageability/dispatcher_state should be removed and is not | systemd-cat
+ ls -l /var/intel-manageability/dispatcher_state
+ cat /var/intel-manageability/dispatcher_state
  exit 1
 fi
 

inbm/integration-reloaded/test/sota/SOTA_UPDATE_SUCCESS_postboot.sh

@@ -32,11 +32,11 @@ echo After reboot, we are looking for exactly 5 files in /var/lib/dispatcher/upl
 ls /var/lib/dispatcher/upload/
 ls /var/lib/dispatcher/upload/ | wc -l | grep 5
 
-echo Also looking for /etc/dispatcher_state to be gone
-if [ -f /etc/dispatcher_state ] ; then
- echo ERROR: /etc/dispatcher_state should be removed and is not | systemd-cat
- ls -l /etc/dispatcher_state
- cat /etc/dispatcher_state
+echo Also looking for /var/intel-manageability/dispatcher_state to be gone
+if [ -f /var/intel-manageability/dispatcher_state ] ; then
+ echo ERROR: /var/intel-manageability/dispatcher_state should be removed and is not | systemd-cat
+ ls -l /var/intel-manageability/dispatcher_state
+ cat /var/intel-manageability/dispatcher_state
  echo "<FAILED> SOTA UPDATE SUCCESS TEST"
  exit 1
 fi

inbm/integration-reloaded/test/sota/SOTA_YOCTO_UPDATE_DOWNLOAD_postboot.sh

@@ -36,9 +36,9 @@ echo After reboot, we are looking for /mender-was-run
 echo After reboot, we are looking for /mender-ext4-was-run
 [ -f "/mender-ext4-was-run" ]
 
-echo Also looking for /etc/dispatcher_state to be gone
-! [ -f /etc/dispatcher_state ]
+echo Also looking for /var/intel-manageability/dispatcher_state to be gone
+! [ -f /var/intel-manageability/dispatcher_state ]
 
-rm -rf /etc/force_yocto /etc/dispatcher_state
+rm -rf /etc/force_yocto /var/intel-manageability/dispatcher_state
 
 cleanup_after_test

inbm/integration-reloaded/test/sota/SOTA_YOCTO_UPDATE_DOWNLOAD_preboot.sh

@@ -17,7 +17,7 @@ mkdir -p /var/log/sota_test
 
 echo "Triggering SOTA integration test: SOTA YOCTO UPDATE DOWNLOAD"
 echo "<START> SOTA YOCTO UPDATE DOWNLOAD" | systemd-cat
-! [ -f /etc/dispatcher_state ]
+! [ -f /var/intel-manageability/dispatcher_state ]
 touch /etc/force_yocto
 ls -l /etc/force_yocto
 date

inbm/packaging/yocto/common/usr.bin.inbm-dispatcher

@@ -48,6 +48,7 @@
  /etc/apt/sources.list rw,
  /etc/apt/sources.list.bak rw,
  /etc/dispatcher_state rw,
+ /var/intel-manageability/dispatcher_state rw,
  /etc/lsb-release r,
  /etc/task_list.yaml r,
  /etc/terminfo/d/dumb r,

inbm/packaging/yocto/meta-intel-ese-manageability/recipes-inb/inb/files/manageability-autoenable

@@ -9,7 +9,7 @@ function run_FOTA() {
  # 2. After that, send mender file and perform SOTA.
  # 3. After system reboot, this script will check if there is SOTA perform previously. If SOTA performed previously and
  # fip binary exist. It will call movisoc tool to flash the fip and reboot.
- if [ -f /etc/dispatcher_state ]; then
+ if [ -f /var/intel-manageability/dispatcher_state ]; then
  if [ -f /var/cache/manageability/repository-tool/fip* ]; then
  echo "Found FIP."
  movisoc-fwu -a /var/cache/manageability/repository-tool/fip*