From f6508206e9f0217c1e86419a2cb518c1b16343fa Mon Sep 17 00:00:00 2001 From: Michael Martin Date: Wed, 5 Jun 2024 16:58:25 -0700 Subject: [PATCH] feat(wasm): add support for wasmtime cache (#12930) (#9389) * feat(wasm): add support for wasmtime cache This adds support for Wasmtime's module caching. See also: * https://github.com/Kong/ngx_wasm_module/pull/540 * https://github.com/Kong/ngx_wasm_module/blob/b19d405403ca6765c548e571010aea3af1accaea/docs/DIRECTIVES.md?plain=1#L136-L149 * https://docs.wasmtime.dev/cli-cache.html * tests(wasm): add start/restart test for wasmtime cache --- .../unreleased/kong/wasm-module-cache.yml | 3 + kong-3.8.0-0.rockspec | 1 + kong/cmd/utils/prefix_handler.lua | 30 ++- kong/conf_loader/init.lua | 6 + kong/templates/nginx.lua | 6 +- kong/templates/wasmtime_cache_config.lua | 17 ++ spec/01-unit/03-conf_loader_spec.lua | 13 ++ spec/01-unit/04-prefix_handler_spec.lua | 8 + .../20-wasm/10-wasmtime_spec.lua | 176 ++++++++++++++++++ spec/fixtures/custom_nginx.template | 6 +- 10 files changed, 263 insertions(+), 3 deletions(-) create mode 100644 changelog/unreleased/kong/wasm-module-cache.yml create mode 100644 kong/templates/wasmtime_cache_config.lua create mode 100644 spec/02-integration/20-wasm/10-wasmtime_spec.lua diff --git a/changelog/unreleased/kong/wasm-module-cache.yml b/changelog/unreleased/kong/wasm-module-cache.yml new file mode 100644 index 000000000000..1b9bd0c8119b --- /dev/null +++ b/changelog/unreleased/kong/wasm-module-cache.yml @@ -0,0 +1,3 @@ +message: Configure Wasmtime module cache when Wasm is enabled +type: feature +scope: Configuration diff --git a/kong-3.8.0-0.rockspec b/kong-3.8.0-0.rockspec index aea5ea012aa2..a04e9e2f8059 100644 --- a/kong-3.8.0-0.rockspec +++ b/kong-3.8.0-0.rockspec @@ -196,6 +196,7 @@ build = { ["kong.templates.nginx_kong_inject"] = "kong/templates/nginx_kong_inject.lua", ["kong.templates.nginx_kong_stream_inject"] = "kong/templates/nginx_kong_stream_inject.lua", ["kong.templates.kong_yml"] = "kong/templates/kong_yml.lua", + ["kong.templates.wasmtime_cache_config"] = "kong/templates/wasmtime_cache_config.lua", ["kong.resty.dns.client"] = "kong/resty/dns/client.lua", ["kong.resty.dns.utils"] = "kong/resty/dns/utils.lua", diff --git a/kong/cmd/utils/prefix_handler.lua b/kong/cmd/utils/prefix_handler.lua index 2f04694b5231..e084c64a53bc 100644 --- a/kong/cmd/utils/prefix_handler.lua +++ b/kong/cmd/utils/prefix_handler.lua @@ -12,6 +12,7 @@ local kong_nginx_stream_template = require "kong.templates.nginx_kong_stream" local nginx_main_inject_template = require "kong.templates.nginx_inject" local nginx_http_inject_template = require "kong.templates.nginx_kong_inject" local nginx_stream_inject_template = require "kong.templates.nginx_kong_stream_inject" +local wasmtime_cache_template = require "kong.templates.wasmtime_cache_config" local system_constants = require "lua_system_constants" local process_secrets = require "kong.cmd.utils.process_secrets" local openssl_bignum = require "resty.openssl.bn" @@ -48,6 +49,7 @@ local math = math local join = pl_path.join local io = io local os = os +local fmt = string.format local function pre_create_private_file(file) @@ -253,6 +255,10 @@ local function get_ulimit() end end +local function quote(s) + return fmt("%q", s) +end + local function compile_conf(kong_config, conf_template, template_env_inject) -- computed config properties for templating local compile_env = { @@ -262,7 +268,8 @@ local function compile_conf(kong_config, conf_template, template_env_inject) tostring = tostring, os = { getenv = os.getenv, - } + }, + quote = quote, } local kong_proxy_access_log = kong_config.proxy_access_log @@ -471,6 +478,10 @@ local function compile_kong_test_inject_conf(kong_config, template, template_env return compile_conf(kong_config, template, template_env) end +local function compile_wasmtime_cache_conf(kong_config) + return compile_conf(kong_config, wasmtime_cache_template) +end + local function prepare_prefixed_interface_dir(usr_path, interface_dir, kong_config) local usr_interface_path = usr_path .. "/" .. interface_dir local interface_path = kong_config.prefix .. "/" .. interface_dir @@ -740,6 +751,23 @@ local function prepare_prefix(kong_config, nginx_custom_template_path, skip_writ return true end + if kong_config.wasm then + if kong_config.wasmtime_cache_directory then + local ok, err = makepath(kong_config.wasmtime_cache_directory) + if not ok then + return nil, err + end + end + + if kong_config.wasmtime_cache_config_file then + local wasmtime_conf, err = compile_wasmtime_cache_conf(kong_config) + if not wasmtime_conf then + return nil, err + end + pl_file.write(kong_config.wasmtime_cache_config_file, wasmtime_conf) + end + end + -- compile Nginx configurations local nginx_template if nginx_custom_template_path then diff --git a/kong/conf_loader/init.lua b/kong/conf_loader/init.lua index ffdee2a570c3..caaa37883394 100644 --- a/kong/conf_loader/init.lua +++ b/kong/conf_loader/init.lua @@ -654,6 +654,12 @@ local function load(path, custom_conf, opts) -- TODO: as a temporary compatibility fix, we are forcing it to 'off'. add_wasm_directive("nginx_http_proxy_wasm_lua_resolver", "off") + -- configure wasmtime module cache + if conf.role == "traditional" or conf.role == "data_plane" then + conf.wasmtime_cache_directory = pl_path.join(conf.prefix, ".wasmtime_cache") + conf.wasmtime_cache_config_file = pl_path.join(conf.prefix, ".wasmtime_config.toml") + end + -- wasm vm properties are inherited from previously set directives if conf.lua_ssl_trusted_certificate and #conf.lua_ssl_trusted_certificate >= 1 then add_wasm_directive("tls_trusted_certificate", conf.lua_ssl_trusted_certificate[1], wasm_main_prefix) diff --git a/kong/templates/nginx.lua b/kong/templates/nginx.lua index 38adba5bbf06..ac8815f7444e 100644 --- a/kong/templates/nginx.lua +++ b/kong/templates/nginx.lua @@ -48,8 +48,12 @@ wasm { > end > end -> if #nginx_wasm_wasmtime_directives > 0 then +> if #nginx_wasm_wasmtime_directives > 0 or wasmtime_cache_config_file then wasmtime { +> if wasmtime_cache_config_file then + cache_config $(quote(wasmtime_cache_config_file)); +> end + > for _, el in ipairs(nginx_wasm_wasmtime_directives) do flag $(el.name) $(el.value); > end diff --git a/kong/templates/wasmtime_cache_config.lua b/kong/templates/wasmtime_cache_config.lua new file mode 100644 index 000000000000..5a70f2f1993c --- /dev/null +++ b/kong/templates/wasmtime_cache_config.lua @@ -0,0 +1,17 @@ +-- This software is copyright Kong Inc. and its licensors. +-- Use of the software is subject to the agreement between your organization +-- and Kong Inc. If there is no such agreement, use is governed by and +-- subject to the terms of the Kong Master Software License Agreement found +-- at https://konghq.com/enterprisesoftwarelicense/. +-- [ END OF LICENSE 0867164ffc95e54f04670b5169c09574bdbd9bba ] + +return [[ +# ************************* +# * DO NOT EDIT THIS FILE * +# ************************* +# This configuration file is auto-generated. +# Any modifications made here will be lost. +[cache] +enabled = true +directory = $(quote(wasmtime_cache_directory)) +]] diff --git a/spec/01-unit/03-conf_loader_spec.lua b/spec/01-unit/03-conf_loader_spec.lua index 07534962ab54..2d2938767f66 100644 --- a/spec/01-unit/03-conf_loader_spec.lua +++ b/spec/01-unit/03-conf_loader_spec.lua @@ -2330,6 +2330,19 @@ describe("Configuration loader", function() assert.is_true(found, "expected the user filter to be enabled") end) + it("populates wasmtime_cache_* properties", function() + local conf, err = conf_loader(nil, { + wasm = "on", + wasm_filters = "bundled,user", + wasm_filters_path = temp_dir, + }) + assert.is_nil(err) + + assert.is_string(conf.wasmtime_cache_directory, + "wasmtime_cache_directory was not set") + assert.is_string(conf.wasmtime_cache_config_file, + "wasmtime_cache_config_file was not set") + end) end) describe("errors", function() diff --git a/spec/01-unit/04-prefix_handler_spec.lua b/spec/01-unit/04-prefix_handler_spec.lua index bf05c78addb0..c17fbd146f34 100644 --- a/spec/01-unit/04-prefix_handler_spec.lua +++ b/spec/01-unit/04-prefix_handler_spec.lua @@ -1087,6 +1087,14 @@ describe("NGINX conf compiler", function() }, debug) ) end) + it("injects wasmtime cache_config", function() + assert.matches( + "wasm {.+wasmtime {.+cache_config .+%.wasmtime_config%.toml.*;", + ngx_cfg({ + wasm = true, + }, debug) + ) + end) describe("injects inherited directives", function() it("only if one isn't explicitly set", function() assert.matches( diff --git a/spec/02-integration/20-wasm/10-wasmtime_spec.lua b/spec/02-integration/20-wasm/10-wasmtime_spec.lua new file mode 100644 index 000000000000..8b1bef2d3858 --- /dev/null +++ b/spec/02-integration/20-wasm/10-wasmtime_spec.lua @@ -0,0 +1,176 @@ +-- This software is copyright Kong Inc. and its licensors. +-- Use of the software is subject to the agreement between your organization +-- and Kong Inc. If there is no such agreement, use is governed by and +-- subject to the terms of the Kong Master Software License Agreement found +-- at https://konghq.com/enterprisesoftwarelicense/. +-- [ END OF LICENSE 0867164ffc95e54f04670b5169c09574bdbd9bba ] + +local helpers = require "spec.helpers" +local fmt = string.format + +for _, role in ipairs({"traditional", "control_plane", "data_plane"}) do + +describe("#wasm wasmtime (role: " .. role .. ")", function() + describe("kong prepare", function() + local conf + local prefix = "./wasm" + + lazy_setup(function() + helpers.clean_prefix(prefix) + assert(helpers.kong_exec("prepare", { + database = role == "data_plane" and "off" or "postgres", + nginx_conf = "spec/fixtures/custom_nginx.template", + wasm = true, + prefix = prefix, + role = role, + cluster_cert = "spec/fixtures/kong_clustering.crt", + cluster_cert_key = "spec/fixtures/kong_clustering.key", + })) + + conf = assert(helpers.get_running_conf(prefix)) + end) + + lazy_teardown(function() + helpers.clean_prefix(prefix) + end) + + if role == "control_plane" then + it("does not populate wasmtime config values", function() + assert.is_nil(conf.wasmtime_cache_directory, + "wasmtime_cache_directory should not be set") + assert.is_nil(conf.wasmtime_cache_config_file, + "wasmtime_cache_config_file should not be set") + end) + + else + it("populates wasmtime config values", function() + assert.is_string(conf.wasmtime_cache_directory, + "wasmtime_cache_directory was not set") + assert.is_string(conf.wasmtime_cache_config_file, + "wasmtime_cache_config_file was not set") + end) + + it("creates the cache directory", function() + assert(helpers.path.isdir(conf.wasmtime_cache_directory), + fmt("expected cache directory (%s) to exist", + conf.wasmtime_cache_directory)) + end) + + it("creates the cache config file", function() + assert(helpers.path.isfile(conf.wasmtime_cache_config_file), + fmt("expected cache config file (%s) to exist", + conf.wasmtime_cache_config_file)) + + local cache_config = assert(helpers.file.read(conf.wasmtime_cache_config_file)) + assert.matches(conf.wasmtime_cache_directory, cache_config, nil, true, + "expected cache config file to reference the cache directory") + end) + end + end) -- kong prepare + + describe("kong stop/start/restart", function() + local conf + local prefix = "./wasm" + local log = prefix .. "/logs/error.log" + local status_port + local client + local cp_prefix = "./wasm-cp" + + lazy_setup(function() + if role == "traditional" then + helpers.get_db_utils("postgres") + end + + helpers.clean_prefix(prefix) + status_port = helpers.get_available_port() + + assert(helpers.kong_exec("prepare", { + database = role == "data_plane" and "off" or "postgres", + nginx_conf = "spec/fixtures/custom_nginx.template", + wasm = true, + prefix = prefix, + role = role, + --wasm_filters_path = helpers.test_conf.wasm_filters_path, + wasm_filters = "tests,response_transformer", + cluster_cert = "spec/fixtures/kong_clustering.crt", + cluster_cert_key = "spec/fixtures/kong_clustering.key", + + status_listen = "127.0.0.1:" .. status_port, + nginx_main_worker_processes = 2, + })) + + conf = assert(helpers.get_running_conf(prefix)) + + -- we need to briefly spin up a control plane, or else we will get + -- error.log entries when our data plane tries to connect + if role == "data_plane" then + helpers.get_db_utils("postgres") + + assert(helpers.start_kong({ + database = "postgres", + nginx_conf = "spec/fixtures/custom_nginx.template", + wasm = true, + prefix = cp_prefix, + role = "control_plane", + wasm_filters = "tests,response_transformer", + cluster_cert = "spec/fixtures/kong_clustering.crt", + cluster_cert_key = "spec/fixtures/kong_clustering.key", + status_listen = "off", + nginx_main_worker_processes = 2, + })) + end + end) + + lazy_teardown(function() + if client then + client:close() + end + + helpers.stop_kong(prefix) + + if role == "data_plane" then + helpers.stop_kong(cp_prefix) + end + end) + + it("does not introduce any errors", function() + local function assert_no_errors() + assert.logfile(log).has.no.line("[error]", true, 0) + assert.logfile(log).has.no.line("[alert]", true, 0) + assert.logfile(log).has.no.line("[emerg]", true, 0) + assert.logfile(log).has.no.line("[crit]", true, 0) + end + + local function assert_kong_status(context) + if not client then + client = helpers.proxy_client(1000, status_port) + client.reopen = true + end + + assert.eventually(function() + local res, err = client:send({ path = "/status", method = "GET" }) + if res and res.status == 200 then + return true + end + + return nil, err or "non-200 status" + end) + .is_truthy("failed waiting for kong status " .. context) + end + + assert(helpers.start_kong(conf, nil, true)) + assert_no_errors() + + assert_kong_status("after fresh startup") + assert_no_errors() + + assert(helpers.restart_kong(conf)) + assert_no_errors() + + assert_kong_status("after restart") + assert_no_errors() + end) + end) -- kong stop/start/restart + +end) -- wasmtime +end -- each role diff --git a/spec/fixtures/custom_nginx.template b/spec/fixtures/custom_nginx.template index 1e56ecbb3136..41aadf7661e6 100644 --- a/spec/fixtures/custom_nginx.template +++ b/spec/fixtures/custom_nginx.template @@ -51,8 +51,12 @@ wasm { > end > end -> if #nginx_wasm_wasmtime_directives > 0 then +> if #nginx_wasm_wasmtime_directives > 0 or wasmtime_cache_config_file then wasmtime { +> if wasmtime_cache_config_file then + cache_config $(quote(wasmtime_cache_config_file)); +> end + > for _, el in ipairs(nginx_wasm_wasmtime_directives) do flag $(el.name) $(el.value); > end