Add internet monitoring (#549)

test/qa/README.md

@@ -28,7 +28,7 @@ source .venv/bin/activate # this should modify your shell prompt
 ```
 3. How to install Python dependencies from file.
 ```bash
-python3 -m pip install -r <path-to-requirements-txt> # requirements.txt is found in `ci/docker/requirements.txt`
+python3 -m pip install -r <path-to-requirements-txt> # requirements.txt is found in `ci/docker/tester/requirements.txt`
 ```
 4. How to install new packages.
 ```bash

test/qa/conftest.py

@@ -0,0 +1,89 @@
+import datetime
+import io
+import threading
+import time
+
+import dns.resolver
+import pytest
+import sh
+
+from lib import logging, network
+
+_CHECK_FREQUENCY=5
+
+def print_to_string(*args, **kwargs):
+    output = io.StringIO()
+    _original_print(*args, file=output, **kwargs)
+    contents = output.getvalue()
+    output.close()
+    return contents
+
+
+_original_print = print
+def _print_with_timestamp(*args, **kwargs):
+    # Get the current time and format it
+    timestamp = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+    # Prepend the timestamp to the original print arguments
+    _original_print(timestamp, *args, **kwargs)
+    logging.log(data=print_to_string(timestamp, *args, **kwargs))
+
+
+# Replace the built-in print with our custom version
+print = _print_with_timestamp # noqa: A001
+
+@pytest.fixture(scope="function", autouse=True)
+def setup_check_internet_connection():
+    print("~~~setup_check_internet_connection: Check internet connection before starting tests")
+    if network.is_available():
+        print("~~~setup_check_internet_connection: BEFORE TEST network.is_available SUCCESS")
+    else:
+        print("~~~setup_check_internet_connection: BEFORE TEST network.is_available FAILURE")
+
+
+@pytest.fixture(scope="session", autouse=True)
+def start_system_monitoring():
+    print("~~~start_system_monitoring: Start system monitoring")
+
+    connection_check_thread = threading.Thread(target=_check_connection_to_ip, args=("1.1.1.1",), daemon=True)
+    connection_out_vpn_check_thread = threading.Thread(target=_check_connection_to_ip_outside_vpn, args=("1.1.1.1",), daemon=True)
+    dns_resolver_thread = threading.Thread(target=_check_dns_resolution, args=("nordvpn.com",), daemon=True)
+    connection_check_thread.start()
+    connection_out_vpn_check_thread.start()
+    dns_resolver_thread.start()
+
+    yield
+
+
+def _check_connection_to_ip(ip_address):
+    while True:
+        try:
+            print(f"~~~_check_connection_to_ip: {ip_address}")
+            "icmp_seq=" in sh.ping("-c", "3", "-W", "3", ip_address) # noqa: B015
+            print(f"~~~_check_connection_to_ip: IN-PING {ip_address} SUCCESS")
+        except sh.ErrorReturnCode as e:
+            print(f"~~~_check_connection_to_ip: IN-PING {ip_address} FAILURE: {e}.")
+        time.sleep(_CHECK_FREQUENCY)
+
+
+def _check_connection_to_ip_outside_vpn(ip_address):
+    while True:
+        try:
+            print(f"~~~_check_connection_to_ip_outside_vpn: {ip_address}")
+            "icmp_seq=" in sh.sudo.ping("-c", "3", "-W", "3", "-m", "57841", ip_address) # noqa: B015
+            print(f"~~~_check_connection_to_ip_outside_vpn: OUT-PING {ip_address} SUCCESS")
+        except sh.ErrorReturnCode as e:
+            print(f"~~~_check_connection_to_ip_outside_vpn: OUT-PING {ip_address} FAILURE: {e}.")
+        time.sleep(_CHECK_FREQUENCY)
+
+
+def _check_dns_resolution(domain):
+    while True:
+        try:
+            print(f"~~~_check_dns_resolution: {domain}")
+            resolver = dns.resolver.Resolver()
+            resolver.nameservers = ['8.8.8.8']
+            resolver.resolve(domain, 'A')  # 'A' for IPv4
+            print(f"~~~_check_dns_resolution: DNS {domain} SUCCESS")
+        except Exception as e:  # noqa: BLE001
+            print(f"~~~_check_dns_resolution: DNS {domain} FAILURE. Error: {e}")
+        time.sleep(_CHECK_FREQUENCY)

test/qa/lib/info.py

@@ -9,7 +9,7 @@ def collect():
     network_interface_info = os.popen("sudo ip addr").read() #sh.sudo.ip.addr()
     routing_info = os.popen("sudo ip route").read() #sh.sudo.ip.route()
     firewall_info = os.popen("sudo iptables -S").read() #sh.sudo.iptables("-S")
-    nameserver_info = os.popen("sudo cat /etc/resolve.conf").read() #sh.sudo.cat("/etc/resolv.conf")
+    nameserver_info = os.popen("sudo cat /etc/resolv.conf").read() #sh.sudo.cat("/etc/resolv.conf")
 
     # without `ww` we cannot see full process lines, as it is cut off early
     processes = sh.ps("-ef", "ww")

test/qa/lib/network.py

@@ -21,6 +21,8 @@
 TSHARK_FILTER_UDP_OBFUSCATED = "udp and (port not 1194) and (ip dst %s)"
 TSHARK_FILTER_TCP_OBFUSCATED = "tcp and (port not 443) and (ip dst %s)"
 
+FWMARK = 57841
+
 class PacketCaptureThread(Thread):
     def __init__(self, connection_settings):
         Thread.__init__(self)
@@ -86,6 +88,18 @@ def _is_internet_reachable(retry=5) -> bool:
     return False
 
 
+def _is_internet_reachable_outside_vpn(retry=5) -> bool:
+    """Returns True when remote host is reachable by its public IP outside VPN tunnel."""
+    i = 0
+    while i < retry:
+        try:
+            return "icmp_seq=" in sh.sudo.ping("-c", "1", "-m", f"{FWMARK}", "-w", "1", "1.1.1.1")
+        except sh.ErrorReturnCode:
+            time.sleep(1)
+            i += 1
+    return False
+
+
 def _is_ipv6_internet_reachable(retry=5) -> bool:
     i = 0
     last = Exception("_is_ipv6_internet_reachable", "error")
@@ -113,6 +127,20 @@ def _is_dns_resolvable(retry=5) -> bool:
     return False
 
 
+def _is_dns_resolvable_outside_vpn(retry: int = 5) -> bool:
+    """Returns True when domain resolution outside vpn is not working."""
+    i = 0
+    while i < retry:
+        try:
+            # @TODO gitlab docker runner has public ipv6, but no connectivity. remove -4 once fixed
+            # @TODO need dns query to go arround vpn tunnel, here with regular ping it does not
+            return "icmp_seq=" in sh.ping("-4", "-c", "1", "-m", f"{FWMARK}", "-w", "1", "nordvpn.com")
+        except sh.ErrorReturnCode:
+            time.sleep(1)
+            i += 1
+    return False
+
+
 def _is_dns_not_resolvable(retry: int = 5) -> bool:
     """Returns True when domain resolution is not working."""
     i = 0
@@ -142,7 +170,9 @@ def is_not_available(retry=5) -> bool:
 
 def is_available(retry=5) -> bool:
     """Returns True when network access is available or throws AssertionError otherwise."""
+    assert _is_internet_reachable_outside_vpn(retry)
     assert _is_internet_reachable(retry)
+    assert _is_dns_resolvable_outside_vpn(retry)
     assert _is_dns_resolvable(retry)
     return True