Merge pull request #58 from cdot65/57-snapshot-error-on-pan-os-1018-m…

…issing-session-counters-in-batch-upgrade

57 snapshot error on pan os 1018 missing session counters in batch upgrade

pan_os_upgrade/models/session_stats.py

@@ -25,7 +25,7 @@ class SessionStats(BaseModel):
     num_gtpu_pending: Optional[str] = Field(..., alias="num-gtpu-pending")
     num_http2_5gc: Optional[str] = Field(..., alias="num-http2-5gc")
     num_icmp: Optional[str] = Field(..., alias="num-icmp")
-    num_imsi: Optional[str] = Field(..., alias="num-imsi")
+    num_imsi: Optional[str] = Field(None, alias="num-imsi")
     num_installed: Optional[str] = Field(..., alias="num-installed")
     num_max: Optional[str] = Field(..., alias="num-max")
     num_mcast: Optional[str] = Field(..., alias="num-mcast")
@@ -36,7 +36,7 @@ class SessionStats(BaseModel):
     num_tcp: Optional[str] = Field(..., alias="num-tcp")
     num_udp: Optional[str] = Field(..., alias="num-udp")
     pps: Optional[str]
-    tcp_cong_ctrl: Optional[str] = Field(..., alias="tcp-cong-ctrl")
+    tcp_cong_ctrl: Optional[str] = Field(None, alias="tcp-cong-ctrl")
     tcp_reject_siw_thresh: Optional[str] = Field(..., alias="tcp-reject-siw-thresh")
     tmo_5gcdelete: Optional[str] = Field(..., alias="tmo-5gcdelete")
     tmo_cp: Optional[str] = Field(..., alias="tmo-cp")

pan_os_upgrade/upgrade.py

@@ -56,6 +56,7 @@
 import time
 import re
 from concurrent.futures import ThreadPoolExecutor, as_completed
+from http.client import RemoteDisconnected
 from logging.handlers import RotatingFileHandler
 from threading import Lock
 from typing import Dict, List, Optional, Tuple, Union
@@ -811,107 +812,62 @@ def perform_reboot(
         # This will initiate a reboot on the device 'device123' and ensure it starts up with version '10.0.1'.
     """
 
-    reboot_start_time = time.time()
     rebooted = False
+    attempt = 0
+    MAX_RETRIES = 30
+    RETRY_DELAY = 60
 
-    # Check if HA details are available
-    if ha_details:
-        logging.info(
-            f"{get_emoji('start')} {hostname}: Rebooting the passive HA target device..."
-        )
-
-    # Reboot standalone target device
-    else:
-        logging.info(
-            f"{get_emoji('start')} {hostname}: Rebooting the standalone target device..."
-        )
+    logging.info(f"{get_emoji('start')} {hostname}: Rebooting the target device...")
 
+    # Initiate reboot
     reboot_job = target_device.op(
-        "<request><restart><system/></restart></request>", cmd_xml=False
+        "<request><restart><system/></restart></request>",
+        cmd_xml=False,
     )
     reboot_job_result = flatten_xml_to_dict(reboot_job)
     logging.info(f"{get_emoji('report')} {hostname}: {reboot_job_result['result']}")
 
     # Wait for the target device reboot process to initiate before checking status
     time.sleep(60)
 
-    # Counter that tracks if the rebooted target device is online but not yet synced on configuration
-    reboot_and_sync_check = 0
-
-    while not rebooted:
-        # Check if HA details are available
-        if ha_details:
-            try:
-                deploy_info, current_ha_details = get_ha_status(
-                    target_device,
-                    hostname,
-                )
-                logging.debug(
-                    f"{get_emoji('report')} {hostname}: deploy_info: {deploy_info}"
-                )
-                logging.debug(
-                    f"{get_emoji('report')} {hostname}: current_ha_details: {current_ha_details}"
-                )
-
-                if current_ha_details and deploy_info in ["active", "passive"]:
-                    if (
-                        current_ha_details["result"]["group"]["running-sync"]
-                        == "synchronized"
-                    ):
-                        logging.info(
-                            f"{get_emoji('success')} {hostname}: HA passive target device rebooted and synchronized with its peer in {int(time.time() - reboot_start_time)} seconds"
-                        )
-                        rebooted = True
-                    else:
-                        reboot_and_sync_check += 1
-                        if reboot_and_sync_check >= 5:
-                            logging.warning(
-                                f"{get_emoji('warning')} {hostname}: HA passive target device rebooted but did not complete a configuration sync with the active after 5 attempts."
-                            )
-                            # Set rebooted to True to exit the loop
-                            rebooted = True
-                            break
-                        else:
-                            logging.info(
-                                f"{get_emoji('working')} {hostname}: HA passive target device rebooted but not yet synchronized with its peer. Will try again in 60 seconds."
-                            )
-                            time.sleep(60)
-            except (PanXapiError, PanConnectionTimeout, PanURLError):
-                logging.info(
-                    f"{get_emoji('working')} {hostname}: Target device is rebooting..."
-                )
-                time.sleep(60)
+    while not rebooted and attempt < MAX_RETRIES:
+        try:
+            # Refresh system information to check if the device is back online
+            target_device.refresh_system_info()
+            current_version = target_device.version
+            logging.info(
+                f"{get_emoji('report')} {hostname}: Current device version: {current_version}"
+            )
 
-        # Reboot standalone target device
-        else:
-            try:
-                target_device.refresh_system_info()
+            # Check if the device has rebooted to the target version
+            if current_version == target_version:
                 logging.info(
-                    f"{get_emoji('report')} {hostname}: Target device version: {target_device.version}"
+                    f"{get_emoji('success')} {hostname}: Device rebooted to the target version successfully."
                 )
-
-                if target_device.version == target_version:
-                    logging.info(
-                        f"{get_emoji('success')} {hostname}: Target device rebooted in {int(time.time() - reboot_start_time)} seconds"
-                    )
-                    rebooted = True
-                else:
-                    logging.error(
-                        f"{get_emoji('stop')} {hostname}: Target device rebooted but running the target version. Please try again."
-                    )
-                    sys.exit(1)
-            except (PanXapiError, PanConnectionTimeout, PanURLError):
-                logging.info(
-                    f"{get_emoji('working')} {hostname}: Target device is rebooting..."
+                rebooted = True
+            else:
+                logging.error(
+                    f"{get_emoji('error')} {hostname}: Device rebooted but not to the target version."
                 )
-                time.sleep(60)
+                sys.exit(1)
 
-        # Check if 30 minutes have passed
-        if time.time() - reboot_start_time > 1800:
-            logging.error(
-                f"{get_emoji('error')} {hostname}: Target device did not become available and/or establish a Connected sync state with its HA peer after 30 minutes. Please check the target device status manually."
+        except (
+            PanXapiError,
+            PanConnectionTimeout,
+            PanURLError,
+            RemoteDisconnected,
+        ) as e:
+            logging.warning(
+                f"{get_emoji('warning')} {hostname}: Retry attempt {attempt + 1} due to error: {e}"
             )
-            break
+            attempt += 1
+            time.sleep(RETRY_DELAY)
+
+    if not rebooted:
+        logging.error(
+            f"{get_emoji('error')} {hostname}: Failed to reboot to the target version after {MAX_RETRIES} attempts."
+        )
+        sys.exit(1)
 
 
 def perform_snapshot(

requirements.txt

@@ -1,39 +1,115 @@
 annotated-types==0.6.0
+appdirs==1.4.3
+argh==0.26.2
 asttokens==2.4.1
+attrs==19.3.0
+Babel==2.14.0
+bandit==1.7.6
 black==23.12.1
+certifi==2023.11.17
 cffi==1.16.0
+chardet==3.0.4
+charset-normalizer==3.3.2
+click==8.1.7
+colorama==0.4.6
+coverage==5.0.3
 cryptography==41.0.7
 decorator==5.1.1
-defusedxml==0.7.1
+dnspython==2.5.0
+eradicate==2.3.0
 executing==2.0.1
+fissix==19.2b1
 flake8==7.0.0
+flake8-bugbear==24.1.17
+flake8-builtins==2.2.0
+flake8-comprehensions==3.14.0
+flake8-docstrings==1.7.0
+flake8-eradicate==1.5.0
+flake8-logging-format==0.9.0
+flake8-mock==0.4
+flake8-mutable==1.2.0
+flake8-pep3101==2.1.0
+flake8-pytest==1.4
+flake8-string-format==0.3.0
+flake8-variables-names==0.0.6
+ghp-import==2.1.0
+gitdb==4.0.11
+GitPython==3.1.41
+idna==3.6
+iniconfig==2.0.0
 ipdb==0.13.13
 ipython==8.20.0
+isort==5.13.2
 jedi==0.19.1
+Jinja2==3.1.3
+livereload==2.6.1
+m2r==0.2.1
+Markdown==3.5.2
+markdown-it-py==3.0.0
+MarkupSafe==2.1.4
 matplotlib-inline==0.1.6
 mccabe==0.7.0
+mdurl==0.1.2
+mergedeep==1.3.4
+mistune==0.8.4
+mkdocs==1.5.3
+mkdocs-autorefs==0.5.0
+mkdocs-material==9.5.4
+mkdocs-material-extensions==1.3.1
+mkdocstrings==0.24.0
+more-itertools==8.1.0
+mypy==1.8.0
 mypy-extensions==1.0.0
 packaging==23.2
+paginate==0.5.6
 pan-os-python==1.11.0
 pan-python==0.17.0
 panos-upgrade-assurance==0.3.1
 parso==0.8.3
 pathspec==0.12.1
+pathtools==0.1.2
+pbr==5.4.4
+pep8-naming==0.13.3
 pexpect==4.9.0
 platformdirs==4.1.0
+pluggy==1.3.0
+poetry-core==1.0.8
+poetry2setup==1.1.0
+port-for==0.3.1
 prompt-toolkit==3.0.43
 ptyprocess==0.7.0
 pure-eval==0.2.2
+py==1.10.0
 pycodestyle==2.11.1
 pycparser==2.21
 pydantic==2.5.3
 pydantic_core==2.14.6
+pydocstyle==4.0.0
 pyflakes==3.2.0
 Pygments==2.17.2
+pymdown-extensions==10.7
 pyOpenSSL==23.3.0
+pytest==7.4.4
+pytest-cov==2.8.1
+python-dateutil==2.8.2
+pytz==2019.3
+PyYAML==6.0.1
+pyyaml_env_tag==0.1
+regex==2023.12.25
+requests==2.31.0
+rich==13.7.0
+shellingham==1.5.4
 six==1.16.0
+smmap==5.0.1
+sphinx-autobuild==0.7.1
+sphinx-rtd-theme==0.4.3
 stack-data==0.6.3
+stevedore==1.31.0
+tornado==6.0.3
 traitlets==5.14.1
+typer==0.9.0
 typing_extensions==4.9.0
+urllib3==2.1.0
+watchdog==3.0.0
 wcwidth==0.2.13
 xmltodict==0.12.0