diff --git a/docker/Dockerfile b/docker/Dockerfile index f3d67b3..86c2beb 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -15,8 +15,8 @@ WORKDIR /app COPY settings.yaml /app # Install any needed packages specified in requirements.txt -# Note: The requirements.txt should contain pan-os-upgrade==1.3.6 -RUN pip install --no-cache-dir pan-os-upgrade==1.3.6 +# Note: The requirements.txt should contain pan-os-upgrade==1.3.7 +RUN pip install --no-cache-dir pan-os-upgrade==1.3.7 # Set the locale to avoid issues with emoji rendering ENV LANG C.UTF-8 diff --git a/docs/about/release-notes.md b/docs/about/release-notes.md index 34d41ff..1532b36 100644 --- a/docs/about/release-notes.md +++ b/docs/about/release-notes.md @@ -2,12 +2,22 @@ Welcome to the release notes for the `pan-os-upgrade` tool. This document provides a detailed record of changes, enhancements, and fixes in each version of the tool. -## Version 1.3.6 +## Version 1.3.7 **Release Date:** *<20240317>* ### What's New in version 1.3.7 +- Added support for active/active firewall pairs. +- Corrected an issue where parsing HA suspended state wasn't converting ElementTree object before conditional matching +- Corrected an issue where suspending of peer HA was not being triggered in certain scenarios + +## Version 1.3.6 + +**Release Date:** *<20240317>* + +### What's New in version 1.3.6 + - Added compatibility check for devices that are in HA to ensure that two minor or major releases aren't being skipped, which will result in a `suspended` state on the firewall that's upgraded first, and then resulting in a broken HA after the second firewall completes its upgrade. ## Version 1.3.5 diff --git a/pan_os_upgrade/components/assurance.py b/pan_os_upgrade/components/assurance.py index 5b5648c..bc1abe8 100644 --- a/pan_os_upgrade/components/assurance.py +++ b/pan_os_upgrade/components/assurance.py @@ -119,7 +119,7 @@ class AssuranceOptions: "enabled_by_default": True, }, "free_disk_space": { - "description": "Check if a there is enough space on the `/opt/panrepo` volume for downloading an PanOS image.", + "description": "Check if a there is enough space on the `/opt/panrepo` volume for PAN-OS image.", "log_level": "warning", "exit_on_failure": False, "enabled_by_default": True, diff --git a/pan_os_upgrade/components/ha.py b/pan_os_upgrade/components/ha.py index ae7bf30..331bc6e 100644 --- a/pan_os_upgrade/components/ha.py +++ b/pan_os_upgrade/components/ha.py @@ -11,6 +11,7 @@ from pan_os_upgrade.components.device import get_ha_status from pan_os_upgrade.components.utilities import ( compare_versions, + flatten_xml_to_dict, get_emoji, ) @@ -272,7 +273,7 @@ def handle_firewall_ha( local_version = ha_details["result"]["group"]["local-info"]["build-rel"] peer_version = ha_details["result"]["group"]["peer-info"]["build-rel"] - if peer_version != local_version: + if ha_details["result"]["group"]["running-sync"] == "synchronized": logging.info( f"HA synchronization complete on {hostname}. Proceeding with upgrade." ) @@ -290,22 +291,44 @@ def handle_firewall_ha( f"{get_emoji(action='report')} {hostname}: Version comparison: {version_comparison}" ) - # If the active and passive target devices are running the same version + # If the firewall and its peer devices are running the same version if version_comparison == "equal": - if local_state == "active": - # Add the active target device to the list and exit the upgrade process + + # if the current device is active or active-primary + if local_state == "active" or local_state == "active-primary": + + # Add the target device to the revisit list and exit the upgrade process with target_devices_to_revisit_lock: target_devices_to_revisit.append(target_device) + + # log message to console logging.info( f"{get_emoji(action='search')} {hostname}: Detected active target device in HA pair running the same version as its peer. Added target device to revisit list." ) + + # Exit the upgrade process for the target device at this time, to be revisited later return False, None - elif local_state == "passive": + # if the current device is passive or active-secondary + elif local_state == "passive" or local_state == "active-secondary": + + # suspend HA state of the target device + if not dry_run: + logging.info( + f"{get_emoji(action='report')} {hostname}: Suspending HA state of passive or active-secondary" + ) + suspend_ha_passive( + target_device, + hostname, + ) + + # log message to console + else: + logging.info( + f"{get_emoji(action='report')} {hostname}: Target device is passive, but we are in dry-run mode. Skipping HA state suspension.", + ) + # Continue with upgrade process on the passive target device - logging.info( - f"{get_emoji(action='report')} {hostname}: Target device is passive", - ) return True, None elif local_state == "initial": @@ -320,9 +343,9 @@ def handle_firewall_ha( f"{get_emoji(action='report')} {hostname}: Target device is on an older version" ) # Suspend HA state of active if the passive is on a later release - if local_state == "active" and not dry_run: + if local_state == "active" or local_state == "active-primary" and not dry_run: logging.info( - f"{get_emoji(action='report')} {hostname}: Suspending HA state of active" + f"{get_emoji(action='report')} {hostname}: Suspending HA state of active or active-primary" ) suspend_ha_active( target_device, @@ -335,9 +358,13 @@ def handle_firewall_ha( f"{get_emoji(action='report')} {hostname}: Target device is on a newer version" ) # Suspend HA state of passive if the active is on a later release - if local_state == "passive" and not dry_run: + if ( + local_state == "passive" + or local_state == "active-secondary" + and not dry_run + ): logging.info( - f"{get_emoji(action='report')} {hostname}: Suspending HA state of passive" + f"{get_emoji(action='report')} {hostname}: Suspending HA state of passive or active-secondary" ) suspend_ha_passive( target_device, @@ -580,7 +607,10 @@ def suspend_ha_active( "", cmd_xml=False, ) - if "success" in suspension_response.text: + + response_message = flatten_xml_to_dict(suspension_response) + + if response_message["result"] == "Successfully changed HA state to suspended": logging.info( f"{get_emoji(action='success')} {hostname}: Active target device HA state suspended." ) @@ -636,12 +666,19 @@ def suspend_ha_passive( - Coordination with network management and understanding the process to resume HA functionality are essential to ensure the continuity of services and network redundancy. """ + logging.info( + f"{get_emoji(action='start')} {hostname}: Suspending passive target device HA state." + ) + try: suspension_response = target_device.op( "", cmd_xml=False, ) - if "success" in suspension_response.text: + + response_message = flatten_xml_to_dict(suspension_response) + + if response_message["result"] == "Successfully changed HA state to suspended": logging.info( f"{get_emoji(action='success')} {hostname}: Passive target device HA state suspended." ) diff --git a/pan_os_upgrade/components/upgrade.py b/pan_os_upgrade/components/upgrade.py index 60818d9..323f5aa 100644 --- a/pan_os_upgrade/components/upgrade.py +++ b/pan_os_upgrade/components/upgrade.py @@ -54,14 +54,14 @@ def check_ha_compatibility( hostname: str, current_major: int, current_minor: int, - upgrade_major: int, - upgrade_minor: int, + target_major: int, + target_minor: int, ) -> bool: """ Checks the compatibility of the target PAN-OS version with the current version in an HA pair. This function assesses whether upgrading a firewall in an HA pair to the target PAN-OS version is compatible - with the current version running on the firewall. It compares the major and minor version numbers to determine + with the current version running on the firewall. It compares the target_major and target_minor version numbers to determine if the upgrade spans more than one major release or if the minor version increment is too large within the same major version. The function logs warnings for potential compatibility issues and returns a boolean indicating whether the upgrade is compatible or not. @@ -76,9 +76,9 @@ def check_ha_compatibility( The current major version number of PAN-OS running on the firewall. current_minor : int The current minor version number of PAN-OS running on the firewall. - upgrade_major : int + target_major : int The target major version number of PAN-OS for the upgrade. - upgrade_minor : int + target_minor : int The target minor version number of PAN-OS for the upgrade. Returns @@ -116,21 +116,21 @@ def check_ha_compatibility( if is_ha_pair: # Check if the major upgrade is more than one release apart - if upgrade_major - current_major > 1: + if target_major - current_major > 1: logging.warning( f"{get_emoji(action='warning')} {hostname}: Upgrading firewalls in an HA pair to a version that is more than one major release apart may cause compatibility issues." ) return False # Check if the upgrade is within the same major version but the minor upgrade is more than one release apart - elif upgrade_major == current_major and upgrade_minor - current_minor > 1: + elif target_major == current_major and target_minor - current_minor > 1: logging.warning( f"{get_emoji(action='warning')} {hostname}: Upgrading firewalls in an HA pair to a version that is more than one minor release apart may cause compatibility issues." ) return False # Check if the upgrade spans exactly one major version but also increases the minor version - elif upgrade_major - current_major == 1 and upgrade_minor > 0: + elif target_major - current_major == 1 and target_minor > 0: logging.warning( f"{get_emoji(action='warning')} {hostname}: Upgrading firewalls in an HA pair to a version that spans more than one major release or increases the minor version beyond the first in the next major release may cause compatibility issues." ) @@ -409,7 +409,7 @@ def software_update_check( settings_file: LazySettings, settings_file_path: Path, target_device: Union[Firewall, Panorama], - version: str, + target_version: str, ) -> bool: """ Checks the availability of the specified software version for upgrade on the target device, taking into account HA configurations. @@ -422,7 +422,7 @@ def software_update_check( The device on which the software version's availability is being checked. hostname : str The hostname or IP address of the target device for identification in logs. - version : str + target_version : str The target PAN-OS version for potential upgrade. ha_details : dict A dictionary containing the HA configuration of the target device, if applicable. @@ -456,39 +456,43 @@ def software_update_check( - Retry logic for downloading the required base image, if not already present, can be customized through the `settings.yaml` file, allowing for operational flexibility and adherence to network policies. """ - # parse version - major, minor, maintenance = version.split(".") + # parse target_version + target_major, target_minor, target_maintenance = target_version.split(".") + + # Convert target_major and target_minor to integers + target_major = int(target_major) + target_minor = int(target_minor) + + # Check if target_maintenance can be converted to an integer + if target_maintenance.isdigit(): + # Convert target_maintenance to integer + target_maintenance = int(target_maintenance) # Make sure we know about the system details - if we have connected via Panorama, this can be null without this. - logging.debug( - f"{get_emoji(action='working')} {hostname}: Refreshing running system information" - ) target_device.refresh_system_info() + current_version = target_device.version # check to see if the specified version is older than the current version determine_upgrade( + current_version=current_version, hostname=hostname, - target_device=target_device, - target_maintenance=maintenance, - target_major=major, - target_minor=minor, + target_maintenance=target_maintenance, + target_major=target_major, + target_minor=target_minor, ) - current_version = target_device.refresh_system_info().version current_parts = current_version.split(".") current_major, current_minor = map(int, current_parts[:2]) - upgrade_parts = version.split(".") - upgrade_major, upgrade_minor = map(int, upgrade_parts[:2]) if ha_details and ha_details["result"].get("enabled"): # Check if the target version is compatible with the current version and the HA setup if not check_ha_compatibility( - ha_details, - hostname, - current_major, - current_minor, - upgrade_major, - upgrade_minor, + ha_details=ha_details, + hostname=hostname, + current_major=current_major, + current_minor=current_minor, + target_major=target_major, + target_minor=target_minor, ): return False @@ -499,24 +503,24 @@ def software_update_check( target_device.software.check() available_versions = target_device.software.versions - if version in available_versions: + if target_version in available_versions: retry_count = settings_file.get("download.max_tries", 3) wait_time = settings_file.get("download.retry_interval", 60) logging.info( - f"{get_emoji(action='success')} {hostname}: version {version} is available for download" + f"{get_emoji(action='success')} {hostname}: version {target_version} is available for download" ) - base_version_key = f"{major}.{minor}.0" + base_version_key = f"{target_major}.{target_minor}.0" if available_versions.get(base_version_key, {}).get("downloaded"): logging.info( - f"{get_emoji(action='success')} {hostname}: Base image for {version} is already downloaded" + f"{get_emoji(action='success')} {hostname}: Base image for {target_version} is already downloaded" ) return True else: for attempt in range(retry_count): logging.error( - f"{get_emoji(action='error')} {hostname}: Base image for {version} is not downloaded. Attempting download." + f"{get_emoji(action='error')} {hostname}: Base image for {target_version} is not downloaded. Attempting download." ) downloaded = software_download( target_device, hostname, base_version_key, ha_details @@ -527,7 +531,7 @@ def software_update_check( f"{get_emoji(action='success')} {hostname}: Base image {base_version_key} downloaded successfully" ) logging.info( - f"{get_emoji(action='success')} {hostname}: Pausing for {wait_time} seconds to let {base_version_key} image load into the software manager before downloading {version}" + f"{get_emoji(action='success')} {hostname}: Pausing for {wait_time} seconds to let {base_version_key} image load into the software manager before downloading {target_version}" ) # Wait before retrying to ensure the device has processed the downloaded base image @@ -535,7 +539,7 @@ def software_update_check( # Re-check the versions after waiting target_device.software.check() - if version in target_device.software.versions: + if target_version in target_device.software.versions: # Proceed with the target version check again return software_update_check( ha_details=ha_details, @@ -543,7 +547,7 @@ def software_update_check( settings_file=settings_file, settings_file_path=settings_file_path, target_device=target_device, - version=version, + target_version=target_version, ) else: @@ -555,7 +559,7 @@ def software_update_check( else: if attempt < retry_count - 1: logging.error( - f"{get_emoji(action='error')} {hostname}: Failed to download base image for version {version}. Retrying in {wait_time} seconds." + f"{get_emoji(action='error')} {hostname}: Failed to download base image for version {target_version}. Retrying in {wait_time} seconds." ) time.sleep(wait_time) else: @@ -566,10 +570,12 @@ def software_update_check( else: # If the version is not available, find and log close matches - close_matches = find_close_matches(list(available_versions.keys()), version) + close_matches = find_close_matches( + list(available_versions.keys()), target_version + ) close_matches_str = ", ".join(close_matches) logging.error( - f"{get_emoji(action='error')} {hostname}: Version {version} is not available for download. Closest matches: {close_matches_str}" + f"{get_emoji(action='error')} {hostname}: Version {target_version} is not available for download. Closest matches: {close_matches_str}" ) return False @@ -690,7 +696,7 @@ def upgrade_firewall( settings_file=settings_file, settings_file_path=settings_file_path, target_device=firewall, - version=target_version, + target_version=target_version, ) # gracefully exit if the firewall is not ready for an upgrade to target version @@ -1014,7 +1020,7 @@ def upgrade_panorama( settings_file=settings_file, settings_file_path=settings_file_path, target_device=panorama, - version=target_version, + target_version=target_version, ) # gracefully exit if the Panorama is not ready for an upgrade to target version diff --git a/pan_os_upgrade/components/utilities.py b/pan_os_upgrade/components/utilities.py index 4bfb4ba..b8c3639 100644 --- a/pan_os_upgrade/components/utilities.py +++ b/pan_os_upgrade/components/utilities.py @@ -481,8 +481,8 @@ def create_firewall_mapping( def determine_upgrade( + current_version: str, hostname: str, - target_device: Union[Firewall, Panorama], target_maintenance: Union[int, str], target_major: int, target_minor: int, @@ -498,9 +498,8 @@ def determine_upgrade( Parameters ---------- - target_device : Union[Firewall, Panorama] - The device (Firewall or Panorama) to be evaluated for an upgrade. This must be an initialized instance with - connectivity to the device. + current_version : str + The device's current PAN-OS version to be evaluated for an upgrade. hostname : str The hostname or IP address of the target device. It is used for logging purposes to clearly identify the device in log messages. @@ -540,7 +539,7 @@ def determine_upgrade( safeguard against unintended firmware changes that could affect device stability and security. """ - current_version = parse_version(version=target_device.version) + current_version_parsed = parse_version(version=current_version) if isinstance(target_maintenance, int): # Handling integer maintenance version separately @@ -552,15 +551,15 @@ def determine_upgrade( ) logging.info( - f"{get_emoji(action='report')} {hostname}: Current version: {target_device.version}" + f"{get_emoji(action='report')} {hostname}: Current version: {current_version}" ) logging.info( f"{get_emoji(action='report')} {hostname}: Target version: {target_major}.{target_minor}.{target_maintenance}" ) - if current_version < target_version: + if current_version_parsed < target_version: logging.info( - f"{get_emoji(action='success')} {hostname}: Upgrade required from {target_device.version} to {target_major}.{target_minor}.{target_maintenance}" + f"{get_emoji(action='success')} {hostname}: Upgrade required from {current_version} to {target_major}.{target_minor}.{target_maintenance}" ) else: logging.info( diff --git a/pan_os_upgrade/main.py b/pan_os_upgrade/main.py index 2f84743..8de0436 100644 --- a/pan_os_upgrade/main.py +++ b/pan_os_upgrade/main.py @@ -586,13 +586,13 @@ def batch( if confirmation: typer.echo(f"{get_emoji(action='start')} Proceeding with the upgrade...") - # Using ThreadPoolExecutor to manage threads + # Setting number of threads for concurrent upgrades threads = SETTINGS_FILE.get("concurrency.threads", 10) logging.info( f"{get_emoji(action='working')} {hostname}: Using {threads} threads." ) - # Using ThreadPoolExecutor to manage threads for upgrading firewalls + # First round of upgrades, targeting all firewalls and placing active firewalls in an HA pair on a revisit list with ThreadPoolExecutor(max_workers=threads) as executor: # Store future objects along with firewalls for reference future_to_firewall = { @@ -619,7 +619,7 @@ def batch( f"{get_emoji(action='error')} {hostname}: Firewall {firewall.hostname} generated an exception: {exc}" ) - # Revisit the firewalls that were skipped in the initial pass + # Second round of upgrades, revisiting firewalls that were active in an HA pair and had the same version as their peers if target_devices_to_revisit: logging.info( f"{get_emoji(action='start')} {hostname}: Revisiting firewalls that were active in an HA pair and had the same version as their peers." diff --git a/pyproject.toml b/pyproject.toml index 371c2b4..08ea0c3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "pan-os-upgrade" -version = "1.3.6" +version = "1.3.7" description = "Python script to automate the upgrade process of PAN-OS firewalls." authors = ["Calvin Remsburg "] documentation = "https://cdot65.github.io/pan-os-upgrade/" diff --git a/tests/test_determine_upgrade.py b/tests/test_determine_upgrade.py index 853c096..e609efd 100644 --- a/tests/test_determine_upgrade.py +++ b/tests/test_determine_upgrade.py @@ -39,11 +39,15 @@ def test_determine_upgrade(): target_minor = 0 # For example, target PAN-OS x.0.x target_maintenance = "1-h1" # For example, target PAN-OS x.x.1-h1 + # Make sure we know about the system details - if we have connected via Panorama, this can be null without this. + target_device.refresh_system_info() + current_version = target_device.version + # Use a try-except block to capture the SystemExit raised by the determine_upgrade function when no upgrade is needed or a downgrade is attempted try: determine_upgrade( + current_version=current_version, hostname=hostname, - target_device=target_device, target_maintenance=target_maintenance, target_major=target_major, target_minor=target_minor, diff --git a/tests/test_get_ha_status.py b/tests/test_get_ha_status.py index 585c3bc..b3a9200 100644 --- a/tests/test_get_ha_status.py +++ b/tests/test_get_ha_status.py @@ -10,8 +10,8 @@ test_cases = [ ("panorama1.cdot.io", "primary-active", None), ("panorama2.cdot.io", "secondary-passive", None), - # ("austin-fw1.cdot.io", "active-primary", None), - # ("austin-fw2.cdot.io", "active-secondary", None), + ("austin-fw1.cdot.io", "active-primary", None), + ("austin-fw2.cdot.io", "active-secondary", None), ("austin-fw3.cdot.io", "disabled", None), ("dallas-fw1.cdot.io", "active", None), ("dallas-fw2.cdot.io", "passive", None), diff --git a/tests/test_handle_firewall_ha.py b/tests/test_handle_firewall_ha.py index fe28816..bd6d8ed 100644 --- a/tests/test_handle_firewall_ha.py +++ b/tests/test_handle_firewall_ha.py @@ -12,6 +12,14 @@ # Define test cases with different HA configurations test_cases = [ + ( + "austin-fw1.cdot.io", + "active-primary", + ), # HA and is active-primary, might be added to revisit list, check proceed accordingly + ( + "austin-fw2.cdot.io", + "active-secondary", + ), # HA and is active, might be added to revisit list, check proceed accordingly ("austin-fw3.cdot.io", None), # Standalone, expecting no HA peer and proceed ( "dallas-fw1.cdot.io",