refactor(pluginservers): code refactor & testing (#12858)

Context
-------

The overall goal of this commit is to refactor the external plugins
implementation, with the following goals in mind:
- Make plugin server code more approachable to unfamiliar engineers and
easier to evolve with confidence
- Harden configuration; ensure configuration defects are caught before
Kong is started
- Extend testing coverage

This is related to ongoing work on the Go PDK, with similar goals in
mind.

Summary
-------

This commit implements the following overall changes to plugin server
code:

- Move configuration related code into conf loader, so that configuration
loading and validation happens at startup time, rather than lazily, when
plugin data is loaded or pluginservers are started. Add tests for
current behavior.

- Move process-management code - for starting up plugin servers as well
as querying external plugins info - into the `process.lua` module.

- Introduce a `kong.runloop.plugin_servers.rpc` module that encapsulates
RPC initialization and protocol-specific implementations. This further
simplifies the main plugin server main module.

- Factor exposed API and phase handlers bridging code into a new `plugin`
module, which encapsulates an external plugin representation, including
the expected fields for any Kong plugin, plus external plugin-specific
bits, such as the RPC instance. Part of this external plugin-specific part
is the instance life cycle management. With this structure, the `kong.runloop.plugin_servers`
main module contains only general external plugin code, including a list
of loaded external plugins, and associated start/stop functions for
plugin servers.

Testing
-------

This commit also implements the following improvements to tests:
- Restructure fixtures to accommodate new external plugin servers --
namely, targeting for now in the existing Python and Javascript
- Add new test cases for external plugins:
  * External plugin configuration: add test cases for current behavior;
    in particular:
    - Fail if no `query_cmd` is provided;
    - Warn if no `start_cmd` is provided - this is by design, as
      external plugins servers can be managed outside of Kong
  * Plugin server start / stop - for both Go and Python plugins
  * External plugin info querying for both Go and Python plugins
  * External plugin execution - for both Go and Python plugins

Internal flow
-------------

`.plugin_servers.init:` loads all external plugins, by calling .plugin_servers.process and `.plugin_servers.plugin`
  `.plugin_servers.process`: queries external plugins info with the command specified in `_query_cmd` proeprties
  `.plugin_servers.plugin`: with info obtained as described above, `.plugin:new` returns a kong-compatible representation
                          of an external plugin, with phase handlers, PRIORITY, and wrappers to the PDK. Calls
                          `.plugin_servers.rpc` to create an RPC through which Kong communicates with the plugin process
    `.plugin_servers.rpc`: based on info contained in the plugin (protocol field), creates the correct RPC for the
                          given external plugin
      `.plugin_servers.rpc.pb_rpc`: protobuf rpc implementation - used by Golang
      `.plugin_servers.rpc.mp.rpc`: messagepack rpc implementation - used by JS and Python
`.plugin_servers.init`: calls `.plugin_servers.process` to start external plugin servers
  `.plugin_servers.process`: optionally starts all external plugin servers (if a `_start_cmd` is found)
     uses the resty pipe API to manage the external plugin process

(cherry picked from commit f88da7d)

.github/workflows/build_and_test.yml

@@ -503,6 +503,8 @@ jobs:
         BUILD_NAME=${{ env.BUILD_NAME_NORMAL }} make dev LIBRARY_PREFIX=${{ env.BUILD_ROOT }}/${{ env.BUILD_NAME_NORMAL }}/kong
         # restore /usr/local folder
         sudo rsync -a --delete /tmp/usr-local-bkp/local/ /usr/local/
+        # python pluginserver tests dependency
+        sudo pip install kong-pdk
 
     - name: Extract FIPS artifact contents
       if: ${{ inputs.aarch64 == false }}

.gitignore

@@ -57,3 +57,4 @@ __pycache__
 bazel-*
 # remove it after migrating from WORKSPACE to Bzlmod
 MODULE.bazel.lock
+spec/fixtures/external_plugins/go/go-hello

kong-3.9.0-0.rockspec

@@ -410,8 +410,11 @@ build = {
     ["kong.runloop.balancer.upstreams"] = "kong/runloop/balancer/upstreams.lua",
     ["kong.runloop.plugin_servers"] = "kong/runloop/plugin_servers/init.lua",
     ["kong.runloop.plugin_servers.process"] = "kong/runloop/plugin_servers/process.lua",
-    ["kong.runloop.plugin_servers.mp_rpc"] = "kong/runloop/plugin_servers/mp_rpc.lua",
-    ["kong.runloop.plugin_servers.pb_rpc"] = "kong/runloop/plugin_servers/pb_rpc.lua",
+    ["kong.runloop.plugin_servers.plugin"] = "kong/runloop/plugin_servers/plugin.lua",
+    ["kong.runloop.plugin_servers.rpc"] = "kong/runloop/plugin_servers/rpc/init.lua",
+    ["kong.runloop.plugin_servers.rpc.util"] = "kong/runloop/plugin_servers/rpc/util.lua",
+    ["kong.runloop.plugin_servers.rpc.mp_rpc"] = "kong/runloop/plugin_servers/rpc/mp_rpc.lua",
+    ["kong.runloop.plugin_servers.rpc.pb_rpc"] = "kong/runloop/plugin_servers/rpc/pb_rpc.lua",
     ["kong.runloop.wasm"] = "kong/runloop/wasm.lua",
     ["kong.runloop.wasm.plugins"] = "kong/runloop/wasm/plugins.lua",
     ["kong.runloop.wasm.properties"] = "kong/runloop/wasm/properties.lua",

kong.conf.default

@@ -176,14 +176,17 @@
 
 #pluginserver_XXX_socket = <prefix>/<XXX>.socket            # Path to the unix socket
                                                             # used by the <XXX> pluginserver.
+
 #pluginserver_XXX_start_cmd = /usr/local/bin/<XXX>          # Full command (including
                                                             # any needed arguments) to
-                                                            # start the <XXX> pluginserver
+                                                            # start the <XXX>
+                                                            # pluginserver.
+
 #pluginserver_XXX_query_cmd = /usr/local/bin/query_<XXX>    # Full command to "query" the
                                                             # <XXX> pluginserver.  Should
                                                             # produce a JSON with the
-                                                            # dump info of all plugins it
-                                                            # manages
+                                                            # dump info of the plugin it
+                                                            # manages.
 
 #port_maps =                     # With this configuration parameter, you can
                                  # let Kong Gateway know the port from

kong/conf_loader/constants.lua

@@ -93,6 +93,9 @@ local DEFAULT_PATHS = {
 }
 
 
+local DEFAULT_PLUGINSERVER_PATH = "/usr/local/bin"
+
+
 local HEADER_KEY_TO_NAME = {
   ["server_tokens"] = "server_tokens",
   ["latency_tokens"] = "latency_tokens",
@@ -664,6 +667,7 @@ return {
 
   CIPHER_SUITES = CIPHER_SUITES,
   DEFAULT_PATHS = DEFAULT_PATHS,
+  DEFAULT_PLUGINSERVER_PATH = DEFAULT_PLUGINSERVER_PATH,
   HEADER_KEY_TO_NAME = HEADER_KEY_TO_NAME,
   UPSTREAM_HEADER_KEY_TO_NAME = UPSTREAM_HEADER_KEY_TO_NAME,
   DYNAMIC_KEY_NAMESPACES = DYNAMIC_KEY_NAMESPACES,

kong/conf_loader/init.lua

@@ -997,6 +997,49 @@ local function load(path, custom_conf, opts)
     conf.active_tracing = false
   end
 
+  -- parse and validate pluginserver directives
+  if conf.pluginserver_names then
+    local pluginservers = {}
+    for i, name in ipairs(conf.pluginserver_names) do
+      name = name:lower()
+      local env_prefix = "pluginserver_" .. name:gsub("-", "_")
+      local socket = conf[env_prefix .. "_socket"] or (conf.prefix .. "/" .. name .. ".socket")
+
+      local start_command = conf[env_prefix .. "_start_cmd"]
+      local query_command = conf[env_prefix .. "_query_cmd"]
+
+      local default_path = exists(conf_constants.DEFAULT_PLUGINSERVER_PATH .. "/" .. name)
+
+      if not start_command and default_path then
+        start_command = default_path
+      end
+
+      if not query_command and default_path then
+        query_command = default_path .. " -dump"
+      end
+
+      -- query_command is required
+      if not query_command then
+        return nil, "query_command undefined for pluginserver " .. name
+      end
+
+      -- if start_command is unset, we assume the pluginserver process is
+      -- managed externally
+      if not start_command then
+        log.warn("start_command undefined for pluginserver " .. name .. "; assuming external process management")
+      end
+
+      pluginservers[i] = {
+        name = name,
+        socket = socket,
+        start_command = start_command,
+        query_command = query_command,
+      }
+    end
+
+    conf.pluginservers = setmetatable(pluginservers, conf_constants._NOP_TOSTRING_MT)
+  end
+
   -- initialize the dns client, so the globally patched tcp.connect method
   -- will work from here onwards.
   assert(require("kong.tools.dns")(conf))