T6362: Create conntrack logger daemon

data/templates/conntrack/sysctl.conf.j2

@@ -6,4 +6,5 @@ net.netfilter.nf_conntrack_max = {{ table_size }}
 net.ipv4.tcp_max_syn_backlog = {{ tcp.half_open_connections }}
 net.netfilter.nf_conntrack_tcp_loose = {{ '1' if tcp.loose is vyos_defined('enable') else '0' }}
 net.netfilter.nf_conntrack_tcp_max_retrans = {{ tcp.max_retrans }}
-net.netfilter.nf_conntrack_acct = {{ '1' if flow_accounting is vyos_defined else '0' }}
+net.netfilter.nf_conntrack_acct = {{ '1' if flow_accounting is vyos_defined else '0' }}
+net.netfilter.nf_conntrack_timestamp = {{ '1' if log.timestamp is vyos_defined else '0' }}

debian/control

@@ -70,6 +70,7 @@ Depends:
   python3-netifaces,
   python3-paramiko,
   python3-passlib,
+  python3-pyroute2,
   python3-psutil,
   python3-pyhumps,
   python3-pystache,

interface-definitions/include/conntrack/log-protocols.xml.i

@@ -0,0 +1,26 @@
+<!-- include start from conntrack/log-protocols.xml.i -->
+<leafNode name="icmp">
+  <properties>
+    <help>Log connection tracking events for ICMP</help>
+    <valueless/>
+  </properties>
+</leafNode>
+<leafNode name="other">
+  <properties>
+    <help>Log connection tracking events for all protocols other than TCP, UDP and ICMP</help>
+    <valueless/>
+  </properties>
+</leafNode>
+<leafNode name="tcp">
+  <properties>
+    <help>Log connection tracking events for TCP</help>
+    <valueless/>
+  </properties>
+</leafNode>
+<leafNode name="udp">
+  <properties>
+    <help>Log connection tracking events for UDP</help>
+    <valueless/>
+  </properties>
+</leafNode>
+<!-- include end -->

interface-definitions/system_conntrack.xml.in

@@ -223,41 +223,59 @@
           </node>
           <node name="log">
             <properties>
-              <help>Log connection tracking events per protocol</help>
+              <help>Log connection tracking</help>
             </properties>
             <children>
-              <node name="icmp">
+              <node name="event">
                 <properties>
-                  <help>Log connection tracking events for ICMP</help>
+                  <help>Event type and protocol</help>
                 </properties>
                 <children>
-                  #include <include/conntrack/log-common.xml.i>
-                </children>
-              </node>
-              <node name="other">
-                <properties>
-                  <help>Log connection tracking events for all protocols other than TCP, UDP and ICMP</help>
-                </properties>
-                <children>
-                  #include <include/conntrack/log-common.xml.i>
+                  <node name="destroy">
+                    <properties>
+                      <help>Log connection deletion</help>
+                    </properties>
+                    <children>
+                      #include <include/conntrack/log-protocols.xml.i>
+                    </children>
+                  </node>
+                  <node name="new">
+                    <properties>
+                      <help>Log connection creation</help>
+                    </properties>
+                    <children>
+                      #include <include/conntrack/log-protocols.xml.i>
+                    </children>
+                  </node>
+                  <node name="update">
+                    <properties>
+                      <help>Log connection updates</help>
+                    </properties>
+                    <children>
+                      #include <include/conntrack/log-protocols.xml.i>
+                    </children>
+                  </node>
                 </children>
               </node>
-              <node name="tcp">
+              <leafNode name="timestamp">
                 <properties>
-                  <help>Log connection tracking events for TCP</help>
+                  <help>Log connection tracking events include flow-based timestamp</help>
+                  <valueless/>
                 </properties>
-                <children>
-                  #include <include/conntrack/log-common.xml.i>
-                </children>
-              </node>
-              <node name="udp">
+              </leafNode>
+              <leafNode name="queue-size">
                 <properties>
-                  <help>Log connection tracking events for UDP</help>
+                  <help>Internal queue size for nf messages</help>
+                  <valueHelp>
+                    <format>u32:100-999999</format>
+                    <description>Number of messages in queue</description>
+                  </valueHelp>
+                  <constraint>
+                    <validator name="numeric" argument="--range 1-999999"/>
+                  </constraint>
+                  <constraintErrorMessage>Queue size must be between 100 and 999999</constraintErrorMessage>
                 </properties>
-                <children>
-                  #include <include/conntrack/log-common.xml.i>
-                </children>
-              </node>
+              </leafNode>
             </children>
           </node>
           <node name="modules">

smoketest/scripts/cli/test_system_conntrack.py

@@ -20,14 +20,17 @@
 from base_vyostest_shim import VyOSUnitTestSHIM
 
 from vyos.firewall import find_nftables_rule
-from vyos.utils.file import read_file
+from vyos.utils.file import read_file, read_json
 
 base_path = ['system', 'conntrack']
 
 def get_sysctl(parameter):
     tmp = parameter.replace(r'.', r'/')
     return read_file(f'/proc/sys/{tmp}')
 
+def get_logger_config():
+    return read_json('/run/vyos-conntrack-logger.conf')
+
 class TestSystemConntrack(VyOSUnitTestSHIM.TestCase):
     @classmethod
     def setUpClass(cls):
@@ -280,5 +283,35 @@ def test_conntrack_timeout_custom(self):
         self.verify_nftables(nftables6_search, 'ip6 vyos_conntrack')
 
         self.cli_delete(['firewall'])
+
+    def test_conntrack_log(self):
+        expected_config = {
+            'event': {
+                'destroy': {},
+                'new': {},
+                'update': {},
+            },
+            'queue_size': '10000'
+        }
+        self.cli_set(base_path + ['log', 'event', 'destroy'])
+        self.cli_set(base_path + ['log', 'event', 'new'])
+        self.cli_set(base_path + ['log', 'event', 'update'])
+        self.cli_set(base_path + ['log', 'queue-size', '10000'])
+        self.cli_commit()
+        self.assertEqual(expected_config, get_logger_config())
+        self.assertEqual('0', get_sysctl('net.netfilter.nf_conntrack_timestamp'))
+
+        for event in ['destroy', 'new', 'update']:
+            for proto in ['icmp', 'other', 'tcp', 'udp']:
+                self.cli_set(base_path + ['log', 'event', event, proto])
+                expected_config['event'][event][proto] = {}
+        self.cli_set(base_path + ['log', 'timestamp'])
+        expected_config['timestamp'] = {}
+        self.cli_commit()
+
+        self.assertEqual(expected_config, get_logger_config())
+        self.assertEqual('1', get_sysctl('net.netfilter.nf_conntrack_timestamp'))
+
+
 if __name__ == '__main__':
     unittest.main(verbosity=2)

src/conf_mode/system_conntrack.py

@@ -13,7 +13,7 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
+import json
 import os
 
 from sys import exit
@@ -24,7 +24,8 @@
 from vyos.utils.dict import dict_search
 from vyos.utils.dict import dict_search_args
 from vyos.utils.dict import dict_search_recursive
-from vyos.utils.process import cmd
+from vyos.utils.file import write_file
+from vyos.utils.process import cmd, call
 from vyos.utils.process import rc_cmd
 from vyos.template import render
 from vyos import ConfigError
@@ -34,6 +35,7 @@
 conntrack_config = r'/etc/modprobe.d/vyatta_nf_conntrack.conf'
 sysctl_file = r'/run/sysctl/10-vyos-conntrack.conf'
 nftables_ct_file = r'/run/nftables-ct.conf'
+vyos_conntrack_logger_config = r'/run/vyos-conntrack-logger.conf'
 
 # Every ALG (Application Layer Gateway) consists of either a Kernel Object
 # also called a Kernel Module/Driver or some rules present in iptables
@@ -113,6 +115,7 @@ def get_config(config=None):
 
     return conntrack
 
+
 def verify(conntrack):
     for inet in ['ipv4', 'ipv6']:
         if dict_search_args(conntrack, 'ignore', inet, 'rule') != None:
@@ -181,6 +184,11 @@ def generate(conntrack):
     if not os.path.exists(nftables_ct_file):
         conntrack['first_install'] = True
 
+    if 'log' not in conntrack:
+        # Remove old conntrack-logger config and return
+        if os.path.exists(vyos_conntrack_logger_config):
+            os.unlink(vyos_conntrack_logger_config)
+
     # Determine if conntrack is needed
     conntrack['ipv4_firewall_action'] = 'return'
     conntrack['ipv6_firewall_action'] = 'return'
@@ -199,6 +207,11 @@ def generate(conntrack):
     render(conntrack_config, 'conntrack/vyos_nf_conntrack.conf.j2', conntrack)
     render(sysctl_file, 'conntrack/sysctl.conf.j2', conntrack)
     render(nftables_ct_file, 'conntrack/nftables-ct.j2', conntrack)
+
+    if 'log' in conntrack:
+        log_conf_json = json.dumps(conntrack['log'], indent=4)
+        write_file(vyos_conntrack_logger_config, log_conf_json)
+
     return None
 
 def apply(conntrack):
@@ -243,8 +256,12 @@ def apply(conntrack):
     # See: https://bugzilla.redhat.com/show_bug.cgi?id=1264080
     cmd(f'sysctl -f {sysctl_file}')
 
+    if 'log' in conntrack:
+        call(f'systemctl restart vyos-conntrack-logger.service')
+
     return None
 
+
 if __name__ == '__main__':
     try:
         c = get_config()