Merge !1450: manager: subprocess debugging via GDB

doc/dev/debugging-with-kresctl.rst

@@ -0,0 +1,109 @@
+.. SPDX-License-Identifier: GPL-3.0-or-later
+
+.. _debugging-with-kresctl:
+
+**********************
+Debugging with kresctl
+**********************
+
+Knot Resolver is made up of several independent components,
+so it can be difficult to debug the individual parts.
+To help with this, there is an option in the kresctl utility
+that can run GDB-compatible debugger on a specific component of the resolver, see the ``debug`` command.
+
+.. program:: kresctl
+
+.. option:: pids
+
+    Lists the PIDs of the Manager's subprocesses, separated by newlines.
+
+    .. option:: --json
+
+        Makes the output more verbose, in JSON. In addition to the subprocesses'
+        PIDs, it also prints their types and statuses.
+
+    .. option:: [proc_type]
+
+        :default: all
+
+        Optional. The type of process to query. See :ref:`Subprocess types
+        <debugging-with-kresctl-subprocess-types>` for more info.
+
+
+.. option:: debug
+
+    Executes a GDB-compatible debugger and attaches it to the Manager's
+    subprocesses. By default, the debugger is ``gdb`` and the subprocesses are
+    only the ``kresd`` workers.
+
+    .. warning::
+
+        The ``debug`` command is a utility for Knot Resolver developers and is
+        not intended to be used by end-users. Running this command **will** make
+        your resolver unresponsive.
+
+    .. note::
+
+        Modern kernels will prevent debuggers from tracing processes that are
+        not their descendants, which is exactly the scenario that happens with
+        ``kresctl debug``. There are three ways to work around this, listed in
+        the order in which they are preferred in terms of security:
+
+          1. Grant the debugger the ``cap_sys_ptrace`` capability
+             (**recommended**)
+
+              * For ``gdb``, this may be achieved by using the ``setcap``
+                command like so:
+
+                .. code-block:: bash
+
+                    sudo setcap cap_sys_ptrace=eip /usr/bin/gdb
+
+          2. Run the debugger as root
+
+              * You may use the ``--sudo`` option to achieve this
+
+          3. Set ``/proc/sys/kernel/yama/ptrace_scope`` to ``0``
+
+              * This will allow **all** programs in your current session to
+                trace each other. Handle with care!
+
+    .. note::
+
+        This command will only work if executed on the same machine where Knot
+        Resolver is running. Remote debugging is currently not supported.
+
+    .. option:: [proc_type]
+
+        :default: kresd
+
+        Optional. The type of process to debug. See :ref:`Subprocess types
+        <debugging-with-kresctl-subprocess-types>` for more info.
+
+    .. option:: --sudo
+
+        Run the debugger with sudo.
+
+    .. option:: --gdb <command>
+
+        Use a custom GDB executable. This may be a command on ``PATH``, or an
+        absolute path to an executable.
+
+    .. option:: --print-only
+
+        Prints the GDB command line into ``stderr`` as a Python array, does not
+        execute GDB.
+
+
+.. _debugging-with-kresctl-subprocess-types:
+
+Subprocess types
+----------------
+
+Some of ``kresctl``'s commands (like :option:`pids` and :option:`debug`) take a subprocess
+type value determining which subprocesses will be affected by them. The possible
+values are as follows:
+
+* ``kresd`` -- the worker daemons
+* ``gc`` -- the cache garbage collector
+* ``all`` -- all of the Manager's subprocesses

doc/dev/index.rst

@@ -30,6 +30,13 @@ Welcome to Knot Resolver's documentation for developers and advanced users!
    manager-dev-code
    layered-protocols
 
+.. toctree::
+   :caption: Debugging
+   :name: debugging-chapter
+   :maxdepth: 1
+
+   debugging-with-kresctl
+
 .. toctree::
    :caption: Lua configuration
    :name: configuration-lua-chapter

poe

@@ -1,4 +1,4 @@
 #!/bin/sh
 
 script_dir="$(dirname "$(readlink -f "$0")")"
-exec poetry --directory "$script_dir" run poe --root "$script_dir" "$@"
+exec poetry --directory "$script_dir" run -- poe --root "$script_dir" "$@"

python/knot_resolver/client/commands/debug.py

@@ -0,0 +1,144 @@
+import argparse
+import json
+import os
+import sys
+from pathlib import Path
+from typing import List, Optional, Tuple, Type
+
+from knot_resolver.client.command import Command, CommandArgs, CompWords, register_command
+from knot_resolver.utils import which
+from knot_resolver.utils.requests import request
+
+PROCS_TYPE = List
+
+
+@register_command
+class DebugCommand(Command):
+    def __init__(self, namespace: argparse.Namespace) -> None:
+        self.proc_type: Optional[str] = namespace.proc_type
+        self.sudo: bool = namespace.sudo
+        self.gdb: str = namespace.gdb
+        self.print_only: bool = namespace.print_only
+        self.gdb_args: List[str] = namespace.extra
+        super().__init__(namespace)
+
+    @staticmethod
+    def register_args_subparser(
+        subparser: "argparse._SubParsersAction[argparse.ArgumentParser]",
+    ) -> Tuple[argparse.ArgumentParser, "Type[Command]"]:
+        debug = subparser.add_parser(
+            "debug",
+            help="Run GDB on the manager's subprocesses",
+        )
+        debug.add_argument(
+            "proc_type",
+            help="Optional, the type of process to debug. May be 'kresd' (default), 'gc', or 'all'.",
+            type=str,
+            nargs="?",
+            default="kresd",
+        )
+        debug.add_argument(
+            "--sudo",
+            dest="sudo",
+            help="Run GDB with sudo",
+            action="store_true",
+            default=False,
+        )
+        debug.add_argument(
+            "--gdb",
+            help="Custom GDB executable (may be a command on PATH, or an absolute path)",
+            type=str,
+            default=None,
+        )
+        debug.add_argument(
+            "--print-only",
+            help="Prints the GDB command line into stderr as a Python array, does not execute GDB",
+            action="store_true",
+            default=False,
+        )
+        return debug, DebugCommand
+
+    @staticmethod
+    def completion(args: List[str], parser: argparse.ArgumentParser) -> CompWords:
+        return {}
+
+    def run(self, args: CommandArgs) -> None:  # noqa: PLR0912, PLR0915
+        if self.gdb is None:
+            try:
+                gdb_cmd = str(which.which("gdb"))
+            except RuntimeError:
+                print("Could not find 'gdb' in $PATH. Is GDB installed?", file=sys.stderr)
+                sys.exit(1)
+        elif "/" not in self.gdb:
+            try:
+                gdb_cmd = str(which.which(self.gdb))
+            except RuntimeError:
+                print(f"Could not find '{self.gdb}' in $PATH.", file=sys.stderr)
+                sys.exit(1)
+        else:
+            gdb_cmd_path = Path(self.gdb).absolute()
+            if not gdb_cmd_path.exists():
+                print(f"Could not find '{self.gdb}'.", file=sys.stderr)
+                sys.exit(1)
+            gdb_cmd = str(gdb_cmd_path)
+
+        response = request(args.socket, "GET", f"processes/{self.proc_type}")
+        if response.status != 200:
+            print(response, file=sys.stderr)
+            sys.exit(1)
+
+        procs = json.loads(response.body)
+        if not isinstance(procs, PROCS_TYPE):
+            print(
+                f"Unexpected response type '{type(procs).__name__}' from manager. Expected '{PROCS_TYPE.__name__}'",
+                file=sys.stderr,
+            )
+            sys.exit(1)
+        if len(procs) == 0:
+            print(
+                f"There are no processes of type '{self.proc_type}' available to debug",
+                file=sys.stderr,
+            )
+
+        exec_args = []
+
+        # Put `sudo --` at the beginning of the command.
+        if self.sudo:
+            try:
+                sudo_cmd = str(which.which("sudo"))
+            except RuntimeError:
+                print("Could not find 'sudo' in $PATH. Is sudo installed?", file=sys.stderr)
+                sys.exit(1)
+            exec_args.extend([sudo_cmd, "--"])
+
+        # Attach GDB to processes - the processes are attached using the `add-inferior` and `attach` GDB
+        # commands. This way, we can debug multiple processes.
+        exec_args.extend([gdb_cmd, "--"])
+        exec_args.extend(["-init-eval-command", "set detach-on-fork off"])
+        exec_args.extend(["-init-eval-command", "set schedule-multiple on"])
+        exec_args.extend(["-init-eval-command", f'attach {procs[0]["pid"]}'])
+        inferior = 2
+        for proc in procs[1:]:
+            exec_args.extend(["-init-eval-command", "add-inferior"])
+            exec_args.extend(["-init-eval-command", f"inferior {inferior}"])
+            exec_args.extend(["-init-eval-command", f'attach {proc["pid"]}'])
+            inferior += 1
+
+        num_inferiors = inferior - 1
+        if num_inferiors > 1:
+            # Now we switch back to the first process and add additional provided GDB arguments.
+            exec_args.extend(["-init-eval-command", "inferior 1"])
+            exec_args.extend(
+                [
+                    "-init-eval-command",
+                    "echo \\n\\nYou are now debugging multiple Knot Resolver processes. To switch between "
+                    "them, use the 'inferior <n>' command, where <n> is an integer from 1 to "
+                    f"{num_inferiors}.\\n\\n",
+                ]
+            )
+        exec_args.extend(self.gdb_args)
+
+        if self.print_only:
+            print(f"{exec_args}")
+        else:
+            os.execl(*exec_args)

python/knot_resolver/client/commands/pids.py

@@ -0,0 +1,63 @@
+import argparse
+import json
+import sys
+from typing import Iterable, List, Optional, Tuple, Type
+
+from knot_resolver.client.command import Command, CommandArgs, CompWords, register_command
+from knot_resolver.utils.requests import request
+
+PROCESSES_TYPE = Iterable
+
+
+@register_command
+class PidsCommand(Command):
+    def __init__(self, namespace: argparse.Namespace) -> None:
+        self.proc_type: Optional[str] = namespace.proc_type
+        self.json: int = namespace.json
+
+        super().__init__(namespace)
+
+    @staticmethod
+    def register_args_subparser(
+        subparser: "argparse._SubParsersAction[argparse.ArgumentParser]",
+    ) -> Tuple[argparse.ArgumentParser, "Type[Command]"]:
+        pids = subparser.add_parser("pids", help="List the PIDs of the Manager's subprocesses")
+        pids.add_argument(
+            "proc_type",
+            help="Optional, the type of process to query. May be 'kresd', 'gc', or 'all' (default).",
+            nargs="?",
+            default="all",
+        )
+        pids.add_argument(
+            "--json",
+            help="Optional, makes the output more verbose, in JSON.",
+            action="store_true",
+            default=False,
+        )
+        return pids, PidsCommand
+
+    @staticmethod
+    def completion(args: List[str], parser: argparse.ArgumentParser) -> CompWords:
+        return {}
+
+    def run(self, args: CommandArgs) -> None:
+        response = request(args.socket, "GET", f"processes/{self.proc_type}")
+
+        if response.status == 200:
+            processes = json.loads(response.body)
+            if isinstance(processes, PROCESSES_TYPE):
+                if self.json:
+                    print(json.dumps(processes, indent=2))
+                else:
+                    for p in processes:
+                        print(p["pid"])
+
+            else:
+                print(
+                    f"Unexpected response type '{type(processes).__name__}' from manager. Expected '{PROCESSES_TYPE.__name__}'",
+                    file=sys.stderr,
+                )
+                sys.exit(1)
+        else:
+            print(response, file=sys.stderr)
+            sys.exit(1)

python/knot_resolver/client/main.py

@@ -1,6 +1,7 @@
 import argparse
 import importlib
 import os
+import sys
 
 from knot_resolver.constants import VERSION
 
@@ -68,7 +69,21 @@ def main() -> None:
     parser = create_main_argument_parser()
     install_commands_parsers(parser)
 
-    namespace = parser.parse_args()
+    # TODO: This is broken with unpatched versions of poethepoet, because they drop the `--` pseudo-argument.
+    # Patch submitted at <https://github.com/nat-n/poethepoet/pull/163>.
+    try:
+        pa_index = sys.argv.index("--", 1)
+        argv_to_parse = sys.argv[1:pa_index]
+        argv_extra = sys.argv[(pa_index + 1) :]
+    except ValueError:
+        argv_to_parse = sys.argv[1:]
+        argv_extra = []
+
+    namespace = parser.parse_args(argv_to_parse)
+    if hasattr(namespace, "extra"):
+        raise TypeError("'extra' is already an attribute - this is disallowed for commands")
+    namespace.extra = argv_extra
+
     client = KresClient(namespace, parser)
     client.execute()
 

python/knot_resolver/controller/interface.py

@@ -109,6 +109,7 @@ def __init__(self, config: KresConfig, kresid: KresID) -> None:
         self._id = kresid
         self._config = config
         self._registered_worker: bool = False
+        self._pid: Optional[int] = None
 
         self._config_file: Optional[Path] = None
         if self.type is SubprocessType.KRESD:
@@ -189,6 +190,10 @@ async def _stop(self) -> None:
     async def _restart(self) -> None:
         pass
 
+    @abstractmethod
+    async def get_pid(self) -> int:
+        pass
+
     @abstractmethod
     def status(self) -> SubprocessStatus:
         pass