From 95a43c35ebae35afa60b010075d5c890375cb2de Mon Sep 17 00:00:00 2001 From: chronolaw Date: Wed, 10 May 2023 13:54:31 +0800 Subject: [PATCH 01/70] validation_tag --- src/ngx_lua_resty_lmdb_module.c | 7 +++++++ src/ngx_lua_resty_lmdb_module.h | 2 ++ 2 files changed, 9 insertions(+) diff --git a/src/ngx_lua_resty_lmdb_module.c b/src/ngx_lua_resty_lmdb_module.c index c371d8ec..3aff321f 100644 --- a/src/ngx_lua_resty_lmdb_module.c +++ b/src/ngx_lua_resty_lmdb_module.c @@ -42,6 +42,13 @@ static ngx_command_t ngx_lua_resty_lmdb_commands[] = { offsetof(ngx_lua_resty_lmdb_conf_t, map_size), NULL }, + { ngx_string("lmdb_validation_tag"), + NGX_MAIN_CONF|NGX_DIRECT_CONF|NGX_CONF_TAKE1, + ngx_conf_set_str_slot, + 0, + offsetof(ngx_lua_resty_lmdb_conf_t, validation_tag), + NULL }, + ngx_null_command }; diff --git a/src/ngx_lua_resty_lmdb_module.h b/src/ngx_lua_resty_lmdb_module.h index 5a7a805d..9cd340ae 100644 --- a/src/ngx_lua_resty_lmdb_module.h +++ b/src/ngx_lua_resty_lmdb_module.h @@ -13,6 +13,8 @@ struct ngx_lua_resty_lmdb_conf_s { size_t map_size; MDB_env *env; MDB_txn *ro_txn; + + ngx_str_t validation_tag; }; From 8d916dec36b281c41f1d68da9f17360f61cbf0a8 Mon Sep 17 00:00:00 2001 From: chronolaw Date: Wed, 10 May 2023 14:48:57 +0800 Subject: [PATCH 02/70] ngx_lua_resty_lmdb_validate --- src/ngx_lua_resty_lmdb_module.c | 71 +++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/src/ngx_lua_resty_lmdb_module.c b/src/ngx_lua_resty_lmdb_module.c index 3aff321f..e7ac4378 100644 --- a/src/ngx_lua_resty_lmdb_module.c +++ b/src/ngx_lua_resty_lmdb_module.c @@ -5,6 +5,10 @@ #define NGX_LUA_RESTY_LMDB_DIR_MODE 0700 +#define NGX_LUA_RESTY_LMDB_VALIDATION_KEY "validation_tag" +#define NGX_LUA_RESTY_LMDB_DEFAULT_DB "_default" + + static ngx_str_t ngx_lua_resty_lmdb_file_names[] = { ngx_string("/data.mdb"), ngx_string("/lock.mdb"), @@ -370,6 +374,68 @@ ngx_lua_resty_lmdb_verify_file_status(ngx_cycle_t *cycle, } +static ngx_int_t +ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, + ngx_lua_resty_lmdb_conf_t *lcf) +{ + int rc; + MDB_dbi dbi; + MDB_val key; + MDB_val value; + MDB_txn *txn = lcf->ro_txn; + + ngx_str_t validation_key = + ngx_string(NGX_LUA_RESTY_LMDB_VALIDATION_KEY); + ngx_str_t default_db = + ngx_string(NGX_LUA_RESTY_LMDB_DEFAULT_DB); + + /* check tag value in lmdb */ + + if (lcf->validation_tag.data == NULL) { + return NGX_OK; + } + + rc = mdb_dbi_open(txn, (const char *) default_db.data, 0, &dbi); + if (rc != 0) { + ngx_log_error(NGX_LOG_ERR, cycle->log, 0, + "unable to open LMDB transaction : %s", + mdb_strerror(rc)); + return NGX_ERROR; + } + + key.mv_size = validation_key.len; + key.mv_data = validation_key.data; + + rc = mdb_get(txn, dbi, &key, &value); + if (rc == 0) { + /* key found, compare with validation_tag value */ + if (ngx_strncmp(lcf->validation_tag.data, + value.mv_data, value.mv_size) == 0) { + + return NGX_OK; + } + + ngx_log_error(NGX_LOG_WARN, cycle->log, 0, + "LMDB validation failed"); + + } else if (rc == MDB_NOTFOUND) { + ngx_log_error(NGX_LOG_WARN, cycle->log, 0, + "LMDB has no validation_tag"); + + } else { + ngx_log_error(NGX_LOG_ERR, cycle->log, 0, + "unable to get LMDB validation_tag : %s", + mdb_strerror(rc)); + mdb_txn_reset(txn); + return NGX_ERROR; + } + + /* drop lmdb db */ + + return NGX_OK; +} + + static ngx_int_t ngx_lua_resty_lmdb_init(ngx_cycle_t *cycle) { ngx_lua_resty_lmdb_conf_t *lcf; @@ -387,6 +453,11 @@ static ngx_int_t ngx_lua_resty_lmdb_init(ngx_cycle_t *cycle) return NGX_ERROR; } + if (ngx_lua_resty_lmdb_validate(cycle, lcf) != NGX_OK) { + ngx_lua_resty_lmdb_close_file(cycle, lcf); + return NGX_ERROR; + } + if (ngx_lua_resty_lmdb_close_file(cycle, lcf) != NGX_OK) { return NGX_ERROR; } From 63a00af234333f9a06e982392a6d3a7f6b222f7e Mon Sep 17 00:00:00 2001 From: chronolaw Date: Wed, 10 May 2023 15:09:53 +0800 Subject: [PATCH 03/70] ngx_lua_resty_lmdb_validate --- src/ngx_lua_resty_lmdb_module.c | 63 +++++++++++++++++++++++++++++++-- 1 file changed, 60 insertions(+), 3 deletions(-) diff --git a/src/ngx_lua_resty_lmdb_module.c b/src/ngx_lua_resty_lmdb_module.c index e7ac4378..bc77da09 100644 --- a/src/ngx_lua_resty_lmdb_module.c +++ b/src/ngx_lua_resty_lmdb_module.c @@ -395,10 +395,18 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, return NGX_OK; } + rc = mdb_txn_renew(txn); + if (rc != 0) { + ngx_log_error(NGX_LOG_ERR, cycle->log, 0, + "unable to renew LMDB transaction : %s", + mdb_strerror(rc)); + return NGX_ERROR; + } + rc = mdb_dbi_open(txn, (const char *) default_db.data, 0, &dbi); if (rc != 0) { ngx_log_error(NGX_LOG_ERR, cycle->log, 0, - "unable to open LMDB transaction : %s", + "unable to open LMDB database : %s", mdb_strerror(rc)); return NGX_ERROR; } @@ -412,6 +420,7 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, if (ngx_strncmp(lcf->validation_tag.data, value.mv_data, value.mv_size) == 0) { + mdb_txn_reset(txn); return NGX_OK; } @@ -426,13 +435,61 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, ngx_log_error(NGX_LOG_ERR, cycle->log, 0, "unable to get LMDB validation_tag : %s", mdb_strerror(rc)); - mdb_txn_reset(txn); - return NGX_ERROR; } + mdb_txn_reset(txn); + /* drop lmdb db */ + rc = mdb_txn_begin(lcf->env, NULL, 0, &txn); + if (rc != 0) { + ngx_log_error(NGX_LOG_CRIT, cycle->log, 0, + "unable to open LMDB write transaction: %s", + mdb_strerror(rc)); + return NGX_ERROR; + } + + rc = mdb_dbi_open(txn, (const char *) default_db.data, 0, &dbi); + if (rc != 0) { + ngx_log_error(NGX_LOG_ERR, cycle->log, 0, + "unable to open LMDB database : %s", + mdb_strerror(rc)); + return NGX_ERROR; + } + + rc = mdb_drop(txn, dbi, 0); + if (rc != 0) { + ngx_log_error(NGX_LOG_ERR, cycle->log, 0, + "unable to drop LMDB database : %s", + mdb_strerror(rc)); + goto failed; + } + + value.mv_size = lcf->validation_tag.len; + value.mv_data = lcf->validation_tag.data; + + rc = mdb_put(txn, dbi, &key, &value, 0); + if (rc != 0) { + ngx_log_error(NGX_LOG_ERR, cycle->log, 0, + "unable to set LMDB validation_tag : %s", + mdb_strerror(rc)); + goto failed; + } + + rc = mdb_txn_commit(txn); + if (rc != 0) { + ngx_log_error(NGX_LOG_ERR, cycle->log, 0, + "unable to commit LMDB : %s", + mdb_strerror(rc)); + goto failed; + } + return NGX_OK; + +failed: + + mdb_txn_abort(txn); + return NGX_ERROR; } From c581ddbfe61fc63f111c9d0c80c08372f5cd6640 Mon Sep 17 00:00:00 2001 From: chronolaw Date: Wed, 10 May 2023 15:22:15 +0800 Subject: [PATCH 04/70] clean --- src/ngx_lua_resty_lmdb_module.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/ngx_lua_resty_lmdb_module.c b/src/ngx_lua_resty_lmdb_module.c index bc77da09..76049d02 100644 --- a/src/ngx_lua_resty_lmdb_module.c +++ b/src/ngx_lua_resty_lmdb_module.c @@ -386,8 +386,6 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, ngx_str_t validation_key = ngx_string(NGX_LUA_RESTY_LMDB_VALIDATION_KEY); - ngx_str_t default_db = - ngx_string(NGX_LUA_RESTY_LMDB_DEFAULT_DB); /* check tag value in lmdb */ @@ -403,11 +401,12 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, return NGX_ERROR; } - rc = mdb_dbi_open(txn, (const char *) default_db.data, 0, &dbi); + rc = mdb_dbi_open(txn, NGX_LUA_RESTY_LMDB_DEFAULT_DB, 0, &dbi); if (rc != 0) { ngx_log_error(NGX_LOG_ERR, cycle->log, 0, "unable to open LMDB database : %s", mdb_strerror(rc)); + mdb_txn_reset(txn); return NGX_ERROR; } @@ -449,7 +448,7 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, return NGX_ERROR; } - rc = mdb_dbi_open(txn, (const char *) default_db.data, 0, &dbi); + rc = mdb_dbi_open(txn, NGX_LUA_RESTY_LMDB_DEFAULT_DB, 0, &dbi); if (rc != 0) { ngx_log_error(NGX_LOG_ERR, cycle->log, 0, "unable to open LMDB database : %s", From e7897636afec28dcfd21dba247159ecead2ae74e Mon Sep 17 00:00:00 2001 From: chronolaw Date: Wed, 10 May 2023 15:27:17 +0800 Subject: [PATCH 05/70] ngx_lua_resty_lmdb_assert --- src/ngx_lua_resty_lmdb_module.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/ngx_lua_resty_lmdb_module.c b/src/ngx_lua_resty_lmdb_module.c index 76049d02..babd54e6 100644 --- a/src/ngx_lua_resty_lmdb_module.c +++ b/src/ngx_lua_resty_lmdb_module.c @@ -393,6 +393,8 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, return NGX_OK; } + ngx_lua_resty_lmdb_assert(txn); + rc = mdb_txn_renew(txn); if (rc != 0) { ngx_log_error(NGX_LOG_ERR, cycle->log, 0, From f6c178de9c7a96acbd0135a37b3dc786511a69d1 Mon Sep 17 00:00:00 2001 From: chronolaw Date: Wed, 10 May 2023 17:53:35 +0800 Subject: [PATCH 06/70] fix crash in cipher --- src/ngx_lua_resty_lmdb_module.c | 63 ++++++++++++++++++++++++++------- 1 file changed, 51 insertions(+), 12 deletions(-) diff --git a/src/ngx_lua_resty_lmdb_module.c b/src/ngx_lua_resty_lmdb_module.c index babd54e6..a39cae34 100644 --- a/src/ngx_lua_resty_lmdb_module.c +++ b/src/ngx_lua_resty_lmdb_module.c @@ -16,6 +16,9 @@ static ngx_str_t ngx_lua_resty_lmdb_file_names[] = { }; +static ngx_cycle_t *ngx_lua_resty_lmdb_init_cycle; + + static void *ngx_lua_resty_lmdb_create_conf(ngx_cycle_t *cycle); static char *ngx_lua_resty_lmdb_init_conf(ngx_cycle_t *cycle, void *conf); static ngx_int_t ngx_lua_resty_lmdb_init(ngx_cycle_t *cycle); @@ -382,7 +385,7 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, MDB_dbi dbi; MDB_val key; MDB_val value; - MDB_txn *txn = lcf->ro_txn; + MDB_txn *txn = NULL; ngx_str_t validation_key = ngx_string(NGX_LUA_RESTY_LMDB_VALIDATION_KEY); @@ -393,22 +396,27 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, return NGX_OK; } - ngx_lua_resty_lmdb_assert(txn); + /* save cycle in init phase for ngx_lua_resty_lmdb_cipher() */ + ngx_lua_resty_lmdb_init_cycle = cycle; - rc = mdb_txn_renew(txn); + rc = mdb_txn_begin(lcf->env, NULL, 0, &txn); if (rc != 0) { - ngx_log_error(NGX_LOG_ERR, cycle->log, 0, - "unable to renew LMDB transaction : %s", + ngx_log_error(NGX_LOG_CRIT, cycle->log, 0, + "unable to open LMDB transaction: %s", mdb_strerror(rc)); + ngx_lua_resty_lmdb_init_cycle = NULL; return NGX_ERROR; } - rc = mdb_dbi_open(txn, NGX_LUA_RESTY_LMDB_DEFAULT_DB, 0, &dbi); + ngx_lua_resty_lmdb_assert(txn); + + rc = mdb_dbi_open(txn, NGX_LUA_RESTY_LMDB_DEFAULT_DB, MDB_CREATE, &dbi); if (rc != 0) { ngx_log_error(NGX_LOG_ERR, cycle->log, 0, "unable to open LMDB database : %s", mdb_strerror(rc)); - mdb_txn_reset(txn); + ngx_lua_resty_lmdb_init_cycle = NULL; + mdb_txn_abort(txn); return NGX_ERROR; } @@ -421,7 +429,8 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, if (ngx_strncmp(lcf->validation_tag.data, value.mv_data, value.mv_size) == 0) { - mdb_txn_reset(txn); + ngx_lua_resty_lmdb_init_cycle = NULL; + mdb_txn_abort(txn); return NGX_OK; } @@ -438,7 +447,7 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, mdb_strerror(rc)); } - mdb_txn_reset(txn); + mdb_txn_abort(txn); /* drop lmdb db */ @@ -447,14 +456,16 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, ngx_log_error(NGX_LOG_CRIT, cycle->log, 0, "unable to open LMDB write transaction: %s", mdb_strerror(rc)); + ngx_lua_resty_lmdb_init_cycle = NULL; return NGX_ERROR; } - rc = mdb_dbi_open(txn, NGX_LUA_RESTY_LMDB_DEFAULT_DB, 0, &dbi); + rc = mdb_dbi_open(txn, NGX_LUA_RESTY_LMDB_DEFAULT_DB, MDB_CREATE, &dbi); if (rc != 0) { ngx_log_error(NGX_LOG_ERR, cycle->log, 0, "unable to open LMDB database : %s", mdb_strerror(rc)); + ngx_lua_resty_lmdb_init_cycle = NULL; return NGX_ERROR; } @@ -466,6 +477,33 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, goto failed; } + rc = mdb_txn_commit(txn); + if (rc != 0) { + ngx_log_error(NGX_LOG_ERR, cycle->log, 0, + "unable to commit LMDB : %s", + mdb_strerror(rc)); + goto failed; + } + + /* set tag to lmdb db */ + + rc = mdb_txn_begin(lcf->env, NULL, 0, &txn); + if (rc != 0) { + ngx_log_error(NGX_LOG_CRIT, cycle->log, 0, + "unable to open LMDB write transaction: %s", + mdb_strerror(rc)); + return NGX_ERROR; + } + + rc = mdb_dbi_open(txn, NGX_LUA_RESTY_LMDB_DEFAULT_DB, MDB_CREATE, &dbi); + if (rc != 0) { + ngx_log_error(NGX_LOG_ERR, cycle->log, 0, + "unable to open LMDB database : %s", + mdb_strerror(rc)); + ngx_lua_resty_lmdb_init_cycle = NULL; + return NGX_ERROR; + } + value.mv_size = lcf->validation_tag.len; value.mv_data = lcf->validation_tag.data; @@ -485,10 +523,13 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, goto failed; } + ngx_lua_resty_lmdb_init_cycle = NULL; + return NGX_OK; failed: + ngx_lua_resty_lmdb_init_cycle = NULL; mdb_txn_abort(txn); return NGX_ERROR; } @@ -564,5 +605,3 @@ static void ngx_lua_resty_lmdb_exit_worker(ngx_cycle_t *cycle) ngx_lua_resty_lmdb_close_file(cycle, lcf); } } - - From 02a107e74082fc29340bff09837267630648bb7a Mon Sep 17 00:00:00 2001 From: chronolaw Date: Wed, 10 May 2023 18:02:49 +0800 Subject: [PATCH 07/70] ngx_lua_resty_lmdb_ngx_cycle --- src/ngx_lua_resty_lmdb_module.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/ngx_lua_resty_lmdb_module.c b/src/ngx_lua_resty_lmdb_module.c index a39cae34..17dbfcd2 100644 --- a/src/ngx_lua_resty_lmdb_module.c +++ b/src/ngx_lua_resty_lmdb_module.c @@ -16,7 +16,7 @@ static ngx_str_t ngx_lua_resty_lmdb_file_names[] = { }; -static ngx_cycle_t *ngx_lua_resty_lmdb_init_cycle; +static ngx_cycle_t *ngx_lua_resty_lmdb_ngx_cycle; static void *ngx_lua_resty_lmdb_create_conf(ngx_cycle_t *cycle); @@ -397,14 +397,14 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, } /* save cycle in init phase for ngx_lua_resty_lmdb_cipher() */ - ngx_lua_resty_lmdb_init_cycle = cycle; + ngx_lua_resty_lmdb_ngx_cycle = cycle; rc = mdb_txn_begin(lcf->env, NULL, 0, &txn); if (rc != 0) { ngx_log_error(NGX_LOG_CRIT, cycle->log, 0, "unable to open LMDB transaction: %s", mdb_strerror(rc)); - ngx_lua_resty_lmdb_init_cycle = NULL; + ngx_lua_resty_lmdb_ngx_cycle = NULL; return NGX_ERROR; } @@ -415,7 +415,7 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, ngx_log_error(NGX_LOG_ERR, cycle->log, 0, "unable to open LMDB database : %s", mdb_strerror(rc)); - ngx_lua_resty_lmdb_init_cycle = NULL; + ngx_lua_resty_lmdb_ngx_cycle = NULL; mdb_txn_abort(txn); return NGX_ERROR; } @@ -429,7 +429,7 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, if (ngx_strncmp(lcf->validation_tag.data, value.mv_data, value.mv_size) == 0) { - ngx_lua_resty_lmdb_init_cycle = NULL; + ngx_lua_resty_lmdb_ngx_cycle = NULL; mdb_txn_abort(txn); return NGX_OK; } @@ -456,7 +456,7 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, ngx_log_error(NGX_LOG_CRIT, cycle->log, 0, "unable to open LMDB write transaction: %s", mdb_strerror(rc)); - ngx_lua_resty_lmdb_init_cycle = NULL; + ngx_lua_resty_lmdb_ngx_cycle = NULL; return NGX_ERROR; } @@ -465,7 +465,7 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, ngx_log_error(NGX_LOG_ERR, cycle->log, 0, "unable to open LMDB database : %s", mdb_strerror(rc)); - ngx_lua_resty_lmdb_init_cycle = NULL; + ngx_lua_resty_lmdb_ngx_cycle = NULL; return NGX_ERROR; } @@ -500,7 +500,7 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, ngx_log_error(NGX_LOG_ERR, cycle->log, 0, "unable to open LMDB database : %s", mdb_strerror(rc)); - ngx_lua_resty_lmdb_init_cycle = NULL; + ngx_lua_resty_lmdb_ngx_cycle = NULL; return NGX_ERROR; } @@ -523,13 +523,13 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, goto failed; } - ngx_lua_resty_lmdb_init_cycle = NULL; + ngx_lua_resty_lmdb_ngx_cycle = NULL; return NGX_OK; failed: - ngx_lua_resty_lmdb_init_cycle = NULL; + ngx_lua_resty_lmdb_ngx_cycle = NULL; mdb_txn_abort(txn); return NGX_ERROR; } From d788efd9f6edb2afa13900656c7e7afd4be52022 Mon Sep 17 00:00:00 2001 From: chronolaw Date: Wed, 10 May 2023 18:08:51 +0800 Subject: [PATCH 08/70] clear init_cycle --- src/ngx_lua_resty_lmdb_module.c | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/src/ngx_lua_resty_lmdb_module.c b/src/ngx_lua_resty_lmdb_module.c index 17dbfcd2..56fe52c0 100644 --- a/src/ngx_lua_resty_lmdb_module.c +++ b/src/ngx_lua_resty_lmdb_module.c @@ -396,15 +396,11 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, return NGX_OK; } - /* save cycle in init phase for ngx_lua_resty_lmdb_cipher() */ - ngx_lua_resty_lmdb_ngx_cycle = cycle; - rc = mdb_txn_begin(lcf->env, NULL, 0, &txn); if (rc != 0) { ngx_log_error(NGX_LOG_CRIT, cycle->log, 0, "unable to open LMDB transaction: %s", mdb_strerror(rc)); - ngx_lua_resty_lmdb_ngx_cycle = NULL; return NGX_ERROR; } @@ -415,7 +411,6 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, ngx_log_error(NGX_LOG_ERR, cycle->log, 0, "unable to open LMDB database : %s", mdb_strerror(rc)); - ngx_lua_resty_lmdb_ngx_cycle = NULL; mdb_txn_abort(txn); return NGX_ERROR; } @@ -429,7 +424,6 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, if (ngx_strncmp(lcf->validation_tag.data, value.mv_data, value.mv_size) == 0) { - ngx_lua_resty_lmdb_ngx_cycle = NULL; mdb_txn_abort(txn); return NGX_OK; } @@ -456,7 +450,6 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, ngx_log_error(NGX_LOG_CRIT, cycle->log, 0, "unable to open LMDB write transaction: %s", mdb_strerror(rc)); - ngx_lua_resty_lmdb_ngx_cycle = NULL; return NGX_ERROR; } @@ -465,7 +458,6 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, ngx_log_error(NGX_LOG_ERR, cycle->log, 0, "unable to open LMDB database : %s", mdb_strerror(rc)); - ngx_lua_resty_lmdb_ngx_cycle = NULL; return NGX_ERROR; } @@ -500,7 +492,6 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, ngx_log_error(NGX_LOG_ERR, cycle->log, 0, "unable to open LMDB database : %s", mdb_strerror(rc)); - ngx_lua_resty_lmdb_ngx_cycle = NULL; return NGX_ERROR; } @@ -523,13 +514,10 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, goto failed; } - ngx_lua_resty_lmdb_ngx_cycle = NULL; - return NGX_OK; failed: - ngx_lua_resty_lmdb_ngx_cycle = NULL; mdb_txn_abort(txn); return NGX_ERROR; } @@ -552,11 +540,17 @@ static ngx_int_t ngx_lua_resty_lmdb_init(ngx_cycle_t *cycle) return NGX_ERROR; } + /* save cycle in init phase for ngx_lua_resty_lmdb_cipher() */ + ngx_lua_resty_lmdb_ngx_cycle = cycle; + if (ngx_lua_resty_lmdb_validate(cycle, lcf) != NGX_OK) { ngx_lua_resty_lmdb_close_file(cycle, lcf); return NGX_ERROR; } + /* set to NULL fro worker processes */ + ngx_lua_resty_lmdb_ngx_cycle = NULL; + if (ngx_lua_resty_lmdb_close_file(cycle, lcf) != NGX_OK) { return NGX_ERROR; } From a2e3a4e4c4020dd0d29f36cc81c77213c830335e Mon Sep 17 00:00:00 2001 From: chronolaw Date: Wed, 10 May 2023 18:13:41 +0800 Subject: [PATCH 09/70] remove some redundant code --- src/ngx_lua_resty_lmdb_module.c | 26 +------------------------- 1 file changed, 1 insertion(+), 25 deletions(-) diff --git a/src/ngx_lua_resty_lmdb_module.c b/src/ngx_lua_resty_lmdb_module.c index 56fe52c0..6b1f85e8 100644 --- a/src/ngx_lua_resty_lmdb_module.c +++ b/src/ngx_lua_resty_lmdb_module.c @@ -469,31 +469,7 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, goto failed; } - rc = mdb_txn_commit(txn); - if (rc != 0) { - ngx_log_error(NGX_LOG_ERR, cycle->log, 0, - "unable to commit LMDB : %s", - mdb_strerror(rc)); - goto failed; - } - - /* set tag to lmdb db */ - - rc = mdb_txn_begin(lcf->env, NULL, 0, &txn); - if (rc != 0) { - ngx_log_error(NGX_LOG_CRIT, cycle->log, 0, - "unable to open LMDB write transaction: %s", - mdb_strerror(rc)); - return NGX_ERROR; - } - - rc = mdb_dbi_open(txn, NGX_LUA_RESTY_LMDB_DEFAULT_DB, MDB_CREATE, &dbi); - if (rc != 0) { - ngx_log_error(NGX_LOG_ERR, cycle->log, 0, - "unable to open LMDB database : %s", - mdb_strerror(rc)); - return NGX_ERROR; - } + /* set tag value to lmdb db */ value.mv_size = lcf->validation_tag.len; value.mv_data = lcf->validation_tag.data; From 43394bc6edb696909789209d392e4678b7c12524 Mon Sep 17 00:00:00 2001 From: chronolaw Date: Thu, 11 May 2023 09:27:29 +0800 Subject: [PATCH 10/70] MDB_CRYPTO_FAIL --- src/ngx_lua_resty_lmdb_module.c | 28 ++++++++-------------------- 1 file changed, 8 insertions(+), 20 deletions(-) diff --git a/src/ngx_lua_resty_lmdb_module.c b/src/ngx_lua_resty_lmdb_module.c index 6b1f85e8..697fcacc 100644 --- a/src/ngx_lua_resty_lmdb_module.c +++ b/src/ngx_lua_resty_lmdb_module.c @@ -411,8 +411,13 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, ngx_log_error(NGX_LOG_ERR, cycle->log, 0, "unable to open LMDB database : %s", mdb_strerror(rc)); - mdb_txn_abort(txn); - return NGX_ERROR; + + if (rc == MDB_CRYPTO_FAIL) { + mdb_txn_abort(txn); + return NGX_DECLINED; + } + + goto failed; } key.mv_size = validation_key.len; @@ -439,28 +444,11 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, ngx_log_error(NGX_LOG_ERR, cycle->log, 0, "unable to get LMDB validation_tag : %s", mdb_strerror(rc)); + goto failed; } - mdb_txn_abort(txn); - /* drop lmdb db */ - rc = mdb_txn_begin(lcf->env, NULL, 0, &txn); - if (rc != 0) { - ngx_log_error(NGX_LOG_CRIT, cycle->log, 0, - "unable to open LMDB write transaction: %s", - mdb_strerror(rc)); - return NGX_ERROR; - } - - rc = mdb_dbi_open(txn, NGX_LUA_RESTY_LMDB_DEFAULT_DB, MDB_CREATE, &dbi); - if (rc != 0) { - ngx_log_error(NGX_LOG_ERR, cycle->log, 0, - "unable to open LMDB database : %s", - mdb_strerror(rc)); - return NGX_ERROR; - } - rc = mdb_drop(txn, dbi, 0); if (rc != 0) { ngx_log_error(NGX_LOG_ERR, cycle->log, 0, From 6aff59d72b70dea42a297240d59d79d79c4fd05b Mon Sep 17 00:00:00 2001 From: chronolaw Date: Thu, 11 May 2023 10:11:14 +0800 Subject: [PATCH 11/70] t/10-validation-tag.t --- t/10-validation-tag.t | 92 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 t/10-validation-tag.t diff --git a/t/10-validation-tag.t b/t/10-validation-tag.t new file mode 100644 index 00000000..83cbe88a --- /dev/null +++ b/t/10-validation-tag.t @@ -0,0 +1,92 @@ +# vim:set ft= ts=4 sw=4 et: + +# run this test after 00-init_mdb.t + +use Test::Nginx::Socket::Lua 'no_plan'; +use Cwd qw(cwd); + +repeat_each(1); + +my $pwd = cwd(); + +our $MainConfig1 = qq{ + lmdb_environment_path /tmp/test10.mdb; + lmdb_map_size 5m; +}; + +our $MainConfig2 = qq{ + lmdb_environment_path /tmp/test10.mdb; + lmdb_map_size 5m; + lmdb_validation_tag 3.3; +}; + +our $HttpConfig = qq{ + lua_package_path "$pwd/lib/?.lua;;"; +}; + +no_long_string(); +#no_diff(); + +no_shuffle(); +run_tests(); + +__DATA__ + +=== TEST 1: no validation_tag +--- http_config eval: $::HttpConfig +--- main_config eval: $::MainConfig1 +--- config + location = /t { + content_by_lua_block { + local l = require("resty.lmdb") + + ngx.say(l.set("test", "value")) + ngx.say(l.get("test")) + ngx.say(l.get("test_not_exist")) + + ngx.say(l.get("validation_tag")) + } + } +--- request +GET /t +--- response_body +true +value +nil +nil +--- no_error_log +[error] +[warn] +[crit] + + +=== TEST 2: start and set validation_tag +--- http_config eval: $::HttpConfig +--- main_config eval: $::MainConfig2 +--- config + location = /t { + content_by_lua_block { + local l = require("resty.lmdb") + + ngx.say(l.get("validation_tag")) + ngx.say(l.get("test")) + + ngx.say(l.set("test", "value")) + ngx.say(l.get("test")) + ngx.say(l.get("test_not_exist")) + } + } +--- request +GET /t +--- response_body +3.3 +nil +true +value +nil +--- error_log +LMDB has no validation_tag +--- no_error_log +[emerg] +[error] +[crit] From 8828ab229108a4dbe6d40061181a1603a043c9b9 Mon Sep 17 00:00:00 2001 From: chronolaw Date: Thu, 11 May 2023 10:22:03 +0800 Subject: [PATCH 12/70] 10-validation-tag-cipher.t --- t/10-validation-tag-cipher.t | 101 ++++++++++++++++++ ...dation-tag.t => 10-validation-tag-plain.t} | 11 +- 2 files changed, 109 insertions(+), 3 deletions(-) create mode 100644 t/10-validation-tag-cipher.t rename t/{10-validation-tag.t => 10-validation-tag-plain.t} (85%) diff --git a/t/10-validation-tag-cipher.t b/t/10-validation-tag-cipher.t new file mode 100644 index 00000000..9263c9a3 --- /dev/null +++ b/t/10-validation-tag-cipher.t @@ -0,0 +1,101 @@ +# vim:set ft= ts=4 sw=4 et: + +# run this test after 00-init_mdb.t + +use Test::Nginx::Socket::Lua; +use Cwd qw(cwd); + +repeat_each(1); + +plan tests => repeat_each() * blocks() * 5 + 1; + +my $pwd = cwd(); + +# remove db for testing +system("rm -rf /tmp/test10-cipher.mdb"); + +our $MainConfig1 = qq{ + lmdb_environment_path /tmp/test10-cipher.mdb; + lmdb_map_size 5m; + lmdb_encryption_key /etc/hostname; + lmdb_encryption_mode "chacha20-poly1305"; +}; + +our $MainConfig2 = qq{ + lmdb_environment_path /tmp/test10-cipher.mdb; + lmdb_map_size 5m; + lmdb_encryption_key /etc/hostname; + lmdb_encryption_mode "chacha20-poly1305"; + lmdb_validation_tag 3.3; +}; + +our $HttpConfig = qq{ + lua_package_path "$pwd/lib/?.lua;;"; +}; + +no_long_string(); +#no_diff(); + +no_shuffle(); +run_tests(); + +__DATA__ + +=== TEST 1: no validation_tag +--- http_config eval: $::HttpConfig +--- main_config eval: $::MainConfig1 +--- config + location = /t { + content_by_lua_block { + local l = require("resty.lmdb") + + ngx.say(l.set("test", "value")) + ngx.say(l.get("test")) + ngx.say(l.get("test_not_exist")) + + ngx.say(l.get("validation_tag")) + } + } +--- request +GET /t +--- response_body +true +value +nil +nil +--- no_error_log +[error] +[warn] +[crit] + + +=== TEST 2: start and set validation_tag +--- http_config eval: $::HttpConfig +--- main_config eval: $::MainConfig2 +--- config + location = /t { + content_by_lua_block { + local l = require("resty.lmdb") + + ngx.say(l.get("validation_tag")) + ngx.say(l.get("test")) + + ngx.say(l.set("test", "value")) + ngx.say(l.get("test")) + ngx.say(l.get("test_not_exist")) + } + } +--- request +GET /t +--- response_body +3.3 +nil +true +value +nil +--- error_log +LMDB has no validation_tag +--- no_error_log +[emerg] +[error] +[crit] diff --git a/t/10-validation-tag.t b/t/10-validation-tag-plain.t similarity index 85% rename from t/10-validation-tag.t rename to t/10-validation-tag-plain.t index 83cbe88a..72360005 100644 --- a/t/10-validation-tag.t +++ b/t/10-validation-tag-plain.t @@ -2,20 +2,25 @@ # run this test after 00-init_mdb.t -use Test::Nginx::Socket::Lua 'no_plan'; +use Test::Nginx::Socket::Lua; use Cwd qw(cwd); repeat_each(1); +plan tests => repeat_each() * blocks() * 5 + 1; + my $pwd = cwd(); +# remove db for testing +system("rm -rf /tmp/test10-plain.mdb"); + our $MainConfig1 = qq{ - lmdb_environment_path /tmp/test10.mdb; + lmdb_environment_path /tmp/test10-plain.mdb; lmdb_map_size 5m; }; our $MainConfig2 = qq{ - lmdb_environment_path /tmp/test10.mdb; + lmdb_environment_path /tmp/test10-plain.mdb; lmdb_map_size 5m; lmdb_validation_tag 3.3; }; From 0eb88fbd9940118ff7ba624c49e3b4d8d5e489a9 Mon Sep 17 00:00:00 2001 From: chronolaw Date: Thu, 11 May 2023 10:25:14 +0800 Subject: [PATCH 13/70] more tests --- t/10-validation-tag-cipher.t | 42 +++++++++++++++++++++++++++++++++++- t/10-validation-tag-plain.t | 40 +++++++++++++++++++++++++++++++++- 2 files changed, 80 insertions(+), 2 deletions(-) diff --git a/t/10-validation-tag-cipher.t b/t/10-validation-tag-cipher.t index 9263c9a3..d7c978a9 100644 --- a/t/10-validation-tag-cipher.t +++ b/t/10-validation-tag-cipher.t @@ -7,7 +7,7 @@ use Cwd qw(cwd); repeat_each(1); -plan tests => repeat_each() * blocks() * 5 + 1; +plan tests => repeat_each() * blocks() * 5 + 2; my $pwd = cwd(); @@ -29,6 +29,14 @@ our $MainConfig2 = qq{ lmdb_validation_tag 3.3; }; +our $MainConfig3 = qq{ + lmdb_environment_path /tmp/test10-cipher.mdb; + lmdb_map_size 5m; + lmdb_encryption_key /etc/hostname; + lmdb_encryption_mode "chacha20-poly1305"; + lmdb_validation_tag 3.4; +}; + our $HttpConfig = qq{ lua_package_path "$pwd/lib/?.lua;;"; }; @@ -99,3 +107,35 @@ LMDB has no validation_tag [emerg] [error] [crit] + + +=== TEST 3: change validation_tag +--- http_config eval: $::HttpConfig +--- main_config eval: $::MainConfig3 +--- config + location = /t { + content_by_lua_block { + local l = require("resty.lmdb") + + ngx.say(l.get("validation_tag")) + ngx.say(l.get("test")) + + ngx.say(l.set("test", "value")) + ngx.say(l.get("test")) + ngx.say(l.get("test_not_exist")) + } + } +--- request +GET /t +--- response_body +3.4 +nil +true +value +nil +--- error_log +LMDB validation failed +--- no_error_log +[emerg] +[error] +[crit] diff --git a/t/10-validation-tag-plain.t b/t/10-validation-tag-plain.t index 72360005..13f6be96 100644 --- a/t/10-validation-tag-plain.t +++ b/t/10-validation-tag-plain.t @@ -7,7 +7,7 @@ use Cwd qw(cwd); repeat_each(1); -plan tests => repeat_each() * blocks() * 5 + 1; +plan tests => repeat_each() * blocks() * 5 + 2; my $pwd = cwd(); @@ -25,6 +25,12 @@ our $MainConfig2 = qq{ lmdb_validation_tag 3.3; }; +our $MainConfig3 = qq{ + lmdb_environment_path /tmp/test10-plain.mdb; + lmdb_map_size 5m; + lmdb_validation_tag 3.4; +}; + our $HttpConfig = qq{ lua_package_path "$pwd/lib/?.lua;;"; }; @@ -95,3 +101,35 @@ LMDB has no validation_tag [emerg] [error] [crit] + + +=== TEST 3: change validation_tag +--- http_config eval: $::HttpConfig +--- main_config eval: $::MainConfig3 +--- config + location = /t { + content_by_lua_block { + local l = require("resty.lmdb") + + ngx.say(l.get("validation_tag")) + ngx.say(l.get("test")) + + ngx.say(l.set("test", "value")) + ngx.say(l.get("test")) + ngx.say(l.get("test_not_exist")) + } + } +--- request +GET /t +--- response_body +3.4 +nil +true +value +nil +--- error_log +LMDB validation failed +--- no_error_log +[emerg] +[error] +[crit] From d5db7fa247f37d9584278707ed2bb464f6d3d1d1 Mon Sep 17 00:00:00 2001 From: chronolaw Date: Thu, 11 May 2023 10:49:04 +0800 Subject: [PATCH 14/70] clean --- src/ngx_lua_resty_lmdb_module.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/ngx_lua_resty_lmdb_module.c b/src/ngx_lua_resty_lmdb_module.c index 697fcacc..08d8f41c 100644 --- a/src/ngx_lua_resty_lmdb_module.c +++ b/src/ngx_lua_resty_lmdb_module.c @@ -411,12 +411,6 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, ngx_log_error(NGX_LOG_ERR, cycle->log, 0, "unable to open LMDB database : %s", mdb_strerror(rc)); - - if (rc == MDB_CRYPTO_FAIL) { - mdb_txn_abort(txn); - return NGX_DECLINED; - } - goto failed; } From 50524ec9d08cfd0355fdc46c44409463ea53f3c4 Mon Sep 17 00:00:00 2001 From: chronolaw Date: Thu, 11 May 2023 15:00:32 +0800 Subject: [PATCH 15/70] typo fix --- src/ngx_lua_resty_lmdb_module.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ngx_lua_resty_lmdb_module.c b/src/ngx_lua_resty_lmdb_module.c index 08d8f41c..11a5485d 100644 --- a/src/ngx_lua_resty_lmdb_module.c +++ b/src/ngx_lua_resty_lmdb_module.c @@ -506,7 +506,7 @@ static ngx_int_t ngx_lua_resty_lmdb_init(ngx_cycle_t *cycle) return NGX_ERROR; } - /* set to NULL fro worker processes */ + /* reset to NULL for worker processes */ ngx_lua_resty_lmdb_ngx_cycle = NULL; if (ngx_lua_resty_lmdb_close_file(cycle, lcf) != NGX_OK) { From e464e6b123ca909186d6c024d8c23d1eb30a5aaa Mon Sep 17 00:00:00 2001 From: chronolaw Date: Mon, 22 May 2023 16:36:06 +0800 Subject: [PATCH 16/70] remove cipher related code --- src/ngx_lua_resty_lmdb_module.c | 9 -- t/10-validation-tag-cipher.t | 141 -------------------------------- 2 files changed, 150 deletions(-) delete mode 100644 t/10-validation-tag-cipher.t diff --git a/src/ngx_lua_resty_lmdb_module.c b/src/ngx_lua_resty_lmdb_module.c index 11a5485d..c1c14877 100644 --- a/src/ngx_lua_resty_lmdb_module.c +++ b/src/ngx_lua_resty_lmdb_module.c @@ -16,9 +16,6 @@ static ngx_str_t ngx_lua_resty_lmdb_file_names[] = { }; -static ngx_cycle_t *ngx_lua_resty_lmdb_ngx_cycle; - - static void *ngx_lua_resty_lmdb_create_conf(ngx_cycle_t *cycle); static char *ngx_lua_resty_lmdb_init_conf(ngx_cycle_t *cycle, void *conf); static ngx_int_t ngx_lua_resty_lmdb_init(ngx_cycle_t *cycle); @@ -498,17 +495,11 @@ static ngx_int_t ngx_lua_resty_lmdb_init(ngx_cycle_t *cycle) return NGX_ERROR; } - /* save cycle in init phase for ngx_lua_resty_lmdb_cipher() */ - ngx_lua_resty_lmdb_ngx_cycle = cycle; - if (ngx_lua_resty_lmdb_validate(cycle, lcf) != NGX_OK) { ngx_lua_resty_lmdb_close_file(cycle, lcf); return NGX_ERROR; } - /* reset to NULL for worker processes */ - ngx_lua_resty_lmdb_ngx_cycle = NULL; - if (ngx_lua_resty_lmdb_close_file(cycle, lcf) != NGX_OK) { return NGX_ERROR; } diff --git a/t/10-validation-tag-cipher.t b/t/10-validation-tag-cipher.t deleted file mode 100644 index d7c978a9..00000000 --- a/t/10-validation-tag-cipher.t +++ /dev/null @@ -1,141 +0,0 @@ -# vim:set ft= ts=4 sw=4 et: - -# run this test after 00-init_mdb.t - -use Test::Nginx::Socket::Lua; -use Cwd qw(cwd); - -repeat_each(1); - -plan tests => repeat_each() * blocks() * 5 + 2; - -my $pwd = cwd(); - -# remove db for testing -system("rm -rf /tmp/test10-cipher.mdb"); - -our $MainConfig1 = qq{ - lmdb_environment_path /tmp/test10-cipher.mdb; - lmdb_map_size 5m; - lmdb_encryption_key /etc/hostname; - lmdb_encryption_mode "chacha20-poly1305"; -}; - -our $MainConfig2 = qq{ - lmdb_environment_path /tmp/test10-cipher.mdb; - lmdb_map_size 5m; - lmdb_encryption_key /etc/hostname; - lmdb_encryption_mode "chacha20-poly1305"; - lmdb_validation_tag 3.3; -}; - -our $MainConfig3 = qq{ - lmdb_environment_path /tmp/test10-cipher.mdb; - lmdb_map_size 5m; - lmdb_encryption_key /etc/hostname; - lmdb_encryption_mode "chacha20-poly1305"; - lmdb_validation_tag 3.4; -}; - -our $HttpConfig = qq{ - lua_package_path "$pwd/lib/?.lua;;"; -}; - -no_long_string(); -#no_diff(); - -no_shuffle(); -run_tests(); - -__DATA__ - -=== TEST 1: no validation_tag ---- http_config eval: $::HttpConfig ---- main_config eval: $::MainConfig1 ---- config - location = /t { - content_by_lua_block { - local l = require("resty.lmdb") - - ngx.say(l.set("test", "value")) - ngx.say(l.get("test")) - ngx.say(l.get("test_not_exist")) - - ngx.say(l.get("validation_tag")) - } - } ---- request -GET /t ---- response_body -true -value -nil -nil ---- no_error_log -[error] -[warn] -[crit] - - -=== TEST 2: start and set validation_tag ---- http_config eval: $::HttpConfig ---- main_config eval: $::MainConfig2 ---- config - location = /t { - content_by_lua_block { - local l = require("resty.lmdb") - - ngx.say(l.get("validation_tag")) - ngx.say(l.get("test")) - - ngx.say(l.set("test", "value")) - ngx.say(l.get("test")) - ngx.say(l.get("test_not_exist")) - } - } ---- request -GET /t ---- response_body -3.3 -nil -true -value -nil ---- error_log -LMDB has no validation_tag ---- no_error_log -[emerg] -[error] -[crit] - - -=== TEST 3: change validation_tag ---- http_config eval: $::HttpConfig ---- main_config eval: $::MainConfig3 ---- config - location = /t { - content_by_lua_block { - local l = require("resty.lmdb") - - ngx.say(l.get("validation_tag")) - ngx.say(l.get("test")) - - ngx.say(l.set("test", "value")) - ngx.say(l.get("test")) - ngx.say(l.get("test_not_exist")) - } - } ---- request -GET /t ---- response_body -3.4 -nil -true -value -nil ---- error_log -LMDB validation failed ---- no_error_log -[emerg] -[error] -[crit] From de0a2a7b1c38a9789883ae794863ef7d4b834e3a Mon Sep 17 00:00:00 2001 From: chronolaw Date: Tue, 23 May 2023 15:45:37 +0800 Subject: [PATCH 17/70] debug log --- src/ngx_lua_resty_lmdb_module.c | 2 ++ t/10-validation-tag-plain.t | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/ngx_lua_resty_lmdb_module.c b/src/ngx_lua_resty_lmdb_module.c index c1c14877..ee1dc1d4 100644 --- a/src/ngx_lua_resty_lmdb_module.c +++ b/src/ngx_lua_resty_lmdb_module.c @@ -390,6 +390,8 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, /* check tag value in lmdb */ if (lcf->validation_tag.data == NULL) { + ngx_log_debug0(NGX_LOG_DEBUG_CORE, cycle->log, 0, + "LMDB validation disabled"); return NGX_OK; } diff --git a/t/10-validation-tag-plain.t b/t/10-validation-tag-plain.t index 13f6be96..45a7b23b 100644 --- a/t/10-validation-tag-plain.t +++ b/t/10-validation-tag-plain.t @@ -7,7 +7,7 @@ use Cwd qw(cwd); repeat_each(1); -plan tests => repeat_each() * blocks() * 5 + 2; +plan tests => repeat_each() * blocks() * 6; my $pwd = cwd(); @@ -65,6 +65,8 @@ true value nil nil +--- error_log +LMDB validation disabled --- no_error_log [error] [warn] From 641cac081374e466064f4bef494e093590b5cb24 Mon Sep 17 00:00:00 2001 From: chronolaw Date: Tue, 30 May 2023 16:38:51 +0800 Subject: [PATCH 18/70] add assert --- src/ngx_lua_resty_lmdb_module.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/ngx_lua_resty_lmdb_module.c b/src/ngx_lua_resty_lmdb_module.c index ee1dc1d4..e94fe5bc 100644 --- a/src/ngx_lua_resty_lmdb_module.c +++ b/src/ngx_lua_resty_lmdb_module.c @@ -395,6 +395,8 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, return NGX_OK; } + ngx_lua_resty_lmdb_assert(lcf->validation_tag.data); + rc = mdb_txn_begin(lcf->env, NULL, 0, &txn); if (rc != 0) { ngx_log_error(NGX_LOG_CRIT, cycle->log, 0, From abcb5a90ed413476710002c658c11f9499314df3 Mon Sep 17 00:00:00 2001 From: chronolaw Date: Tue, 4 Jul 2023 09:20:12 +0800 Subject: [PATCH 19/70] ngx_lua_resty_lmdb_write_tag --- src/ngx_lua_resty_lmdb_module.c | 38 +++++++++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/src/ngx_lua_resty_lmdb_module.c b/src/ngx_lua_resty_lmdb_module.c index e94fe5bc..442c8511 100644 --- a/src/ngx_lua_resty_lmdb_module.c +++ b/src/ngx_lua_resty_lmdb_module.c @@ -439,21 +439,51 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, ngx_log_error(NGX_LOG_ERR, cycle->log, 0, "unable to get LMDB validation_tag : %s", mdb_strerror(rc)); - goto failed; } - /* drop lmdb db */ + mdb_txn_abort(txn); + return NGX_ERROR; +} + - rc = mdb_drop(txn, dbi, 0); +static ngx_int_t +ngx_lua_resty_lmdb_write_tag(ngx_cycle_t *cycle, + ngx_lua_resty_lmdb_conf_t *lcf) +{ + int rc; + MDB_dbi dbi; + MDB_val key; + MDB_val value; + MDB_txn *txn = NULL; + + ngx_str_t validation_key = + ngx_string(NGX_LUA_RESTY_LMDB_VALIDATION_KEY); + + ngx_lua_resty_lmdb_assert(lcf->validation_tag.data); + + rc = mdb_txn_begin(lcf->env, NULL, 0, &txn); + if (rc != 0) { + ngx_log_error(NGX_LOG_CRIT, cycle->log, 0, + "unable to open LMDB transaction: %s", + mdb_strerror(rc)); + return NGX_ERROR; + } + + ngx_lua_resty_lmdb_assert(txn); + + rc = mdb_dbi_open(txn, NGX_LUA_RESTY_LMDB_DEFAULT_DB, MDB_CREATE, &dbi); if (rc != 0) { ngx_log_error(NGX_LOG_ERR, cycle->log, 0, - "unable to drop LMDB database : %s", + "unable to open LMDB database : %s", mdb_strerror(rc)); goto failed; } /* set tag value to lmdb db */ + key.mv_size = validation_key.len; + key.mv_data = validation_key.data; + value.mv_size = lcf->validation_tag.len; value.mv_data = lcf->validation_tag.data; From f7847d01a693fb67f00940eb6b7eb0d0df9d8fbf Mon Sep 17 00:00:00 2001 From: chronolaw Date: Tue, 4 Jul 2023 09:34:55 +0800 Subject: [PATCH 20/70] remove if validation failed --- src/ngx_lua_resty_lmdb_module.c | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/src/ngx_lua_resty_lmdb_module.c b/src/ngx_lua_resty_lmdb_module.c index 442c8511..e140d37c 100644 --- a/src/ngx_lua_resty_lmdb_module.c +++ b/src/ngx_lua_resty_lmdb_module.c @@ -441,6 +441,8 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, mdb_strerror(rc)); } +failed: + mdb_txn_abort(txn); return NGX_ERROR; } @@ -529,9 +531,26 @@ static ngx_int_t ngx_lua_resty_lmdb_init(ngx_cycle_t *cycle) return NGX_ERROR; } + /* check lmdb validation tag */ + if (ngx_lua_resty_lmdb_validate(cycle, lcf) != NGX_OK) { ngx_lua_resty_lmdb_close_file(cycle, lcf); - return NGX_ERROR; + + /* remove lmdb files to clear data */ + if (ngx_lua_resty_lmdb_remove_files(cycle, lcf->env_path) != NGX_OK) { + return NGX_ERROR; + } + + /* open lmdb file again */ + if (ngx_lua_resty_lmdb_open_file(cycle, lcf, 1) != NGX_OK) { + return NGX_ERROR; + } + + /* write tag into lmdb */ + if (ngx_lua_resty_lmdb_write_tag(cycle, lcf) != NGX_OK) { + ngx_lua_resty_lmdb_close_file(cycle, lcf); + return NGX_ERROR; + } } if (ngx_lua_resty_lmdb_close_file(cycle, lcf) != NGX_OK) { From 41a1eae4d03f6d4d1d3243d46ddd07295860ed6a Mon Sep 17 00:00:00 2001 From: chronolaw Date: Fri, 14 Jul 2023 14:09:15 +0800 Subject: [PATCH 21/70] style lint --- src/ngx_lua_resty_lmdb_module.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/ngx_lua_resty_lmdb_module.c b/src/ngx_lua_resty_lmdb_module.c index e140d37c..3d56f20e 100644 --- a/src/ngx_lua_resty_lmdb_module.c +++ b/src/ngx_lua_resty_lmdb_module.c @@ -601,3 +601,5 @@ static void ngx_lua_resty_lmdb_exit_worker(ngx_cycle_t *cycle) ngx_lua_resty_lmdb_close_file(cycle, lcf); } } + + From 9759cd7130ba34d752e235a0436bc0a981bea73a Mon Sep 17 00:00:00 2001 From: chronolaw Date: Thu, 24 Aug 2023 11:19:50 +0800 Subject: [PATCH 22/70] change ngx_lua_resty_lmdb_remove_files --- src/ngx_lua_resty_lmdb_module.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/ngx_lua_resty_lmdb_module.c b/src/ngx_lua_resty_lmdb_module.c index 3d56f20e..947bed30 100644 --- a/src/ngx_lua_resty_lmdb_module.c +++ b/src/ngx_lua_resty_lmdb_module.c @@ -161,13 +161,14 @@ ngx_lua_resty_lmdb_create_env(ngx_cycle_t *cycle, static ngx_int_t -ngx_lua_resty_lmdb_remove_files(ngx_cycle_t *cycle, ngx_path_t *path) +ngx_lua_resty_lmdb_remove_files(ngx_cycle_t *cycle, ngx_lua_resty_lmdb_conf_t *lcf) { ngx_file_info_t fi; u_char name_buf[NGX_MAX_PATH]; ngx_str_t *names = ngx_lua_resty_lmdb_file_names; ngx_str_t *name; + ngx_path_t *path = lcf->env_path; ngx_log_error(NGX_LOG_WARN, cycle->log, 0, "LMDB database is corrupted or incompatible, removing"); @@ -244,7 +245,7 @@ ngx_lua_resty_lmdb_open_file(ngx_cycle_t *cycle, mdb_env_close(lcf->env); lcf->env = NULL; - if (ngx_lua_resty_lmdb_remove_files(cycle, lcf->env_path) != NGX_OK) { + if (ngx_lua_resty_lmdb_remove_files(cycle, lcf) != NGX_OK) { return NGX_ERROR; } @@ -537,7 +538,7 @@ static ngx_int_t ngx_lua_resty_lmdb_init(ngx_cycle_t *cycle) ngx_lua_resty_lmdb_close_file(cycle, lcf); /* remove lmdb files to clear data */ - if (ngx_lua_resty_lmdb_remove_files(cycle, lcf->env_path) != NGX_OK) { + if (ngx_lua_resty_lmdb_remove_files(cycle, lcf) != NGX_OK) { return NGX_ERROR; } From fe987157f84a88aafcb1b1a3a8da8039827b68b3 Mon Sep 17 00:00:00 2001 From: chronolaw Date: Sun, 3 Sep 2023 17:15:34 +0800 Subject: [PATCH 23/70] compare value.mv_size --- src/ngx_lua_resty_lmdb_module.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ngx_lua_resty_lmdb_module.c b/src/ngx_lua_resty_lmdb_module.c index 947bed30..7796aa79 100644 --- a/src/ngx_lua_resty_lmdb_module.c +++ b/src/ngx_lua_resty_lmdb_module.c @@ -422,7 +422,8 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, rc = mdb_get(txn, dbi, &key, &value); if (rc == 0) { /* key found, compare with validation_tag value */ - if (ngx_strncmp(lcf->validation_tag.data, + if (lcf->validation_tag.len == value.mv_size && + ngx_strncmp(lcf->validation_tag.data, value.mv_data, value.mv_size) == 0) { mdb_txn_abort(txn); From 3ae4cf3d72075a94e7ddf6f03dfeb7cb40d5174f Mon Sep 17 00:00:00 2001 From: chronolaw Date: Wed, 13 Sep 2023 14:00:45 +0800 Subject: [PATCH 24/70] readme --- README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/README.md b/README.md index c0e5f238..b40046fd 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,7 @@ Table of Contents * [lmdb_environment_path](#lmdb_environment_path) * [lmdb_max_databases](#lmdb_max_databases) * [lmdb_map_size](#lmdb_map_size) + * [lmdb_validation_tag](#lmdb_validation_tag) * [Copyright and license](#copyright-and-license) ## APIs @@ -228,6 +229,19 @@ Set the size of the memory map, the default value is `1048576`(1MB). [Back to TOC](#table-of-contents) +### lmdb_validation_tag + +**syntax:** *lmdb_validation_tag value;* + +**context:** *main* + +Set a content validation tag into LMDB. +When LMDB starts, it will check the tag value, +if the value is different from the directive value, +the content of LMDB will be cleaned up. + +[Back to TOC](#table-of-contents) + ## Copyright and license Copyright (c) 2021-2022 Kong, Inc. From 02f1eed5ec9cd12ce0660720881ef6ae51d0cc51 Mon Sep 17 00:00:00 2001 From: chronolaw Date: Wed, 13 Sep 2023 14:04:20 +0800 Subject: [PATCH 25/70] typo fix --- src/ngx_lua_resty_lmdb_module.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ngx_lua_resty_lmdb_module.c b/src/ngx_lua_resty_lmdb_module.c index 7796aa79..02c93ef7 100644 --- a/src/ngx_lua_resty_lmdb_module.c +++ b/src/ngx_lua_resty_lmdb_module.c @@ -538,7 +538,7 @@ static ngx_int_t ngx_lua_resty_lmdb_init(ngx_cycle_t *cycle) if (ngx_lua_resty_lmdb_validate(cycle, lcf) != NGX_OK) { ngx_lua_resty_lmdb_close_file(cycle, lcf); - /* remove lmdb files to clear data */ + /* remove lmdb files to clean data */ if (ngx_lua_resty_lmdb_remove_files(cycle, lcf) != NGX_OK) { return NGX_ERROR; } From db76a574831b6c0e1b844808d0ccbdf674cf3e29 Mon Sep 17 00:00:00 2001 From: chronolaw Date: Wed, 10 May 2023 13:54:31 +0800 Subject: [PATCH 26/70] validation_tag --- src/ngx_lua_resty_lmdb_module.c | 7 +++++++ src/ngx_lua_resty_lmdb_module.h | 2 ++ 2 files changed, 9 insertions(+) diff --git a/src/ngx_lua_resty_lmdb_module.c b/src/ngx_lua_resty_lmdb_module.c index c371d8ec..3aff321f 100644 --- a/src/ngx_lua_resty_lmdb_module.c +++ b/src/ngx_lua_resty_lmdb_module.c @@ -42,6 +42,13 @@ static ngx_command_t ngx_lua_resty_lmdb_commands[] = { offsetof(ngx_lua_resty_lmdb_conf_t, map_size), NULL }, + { ngx_string("lmdb_validation_tag"), + NGX_MAIN_CONF|NGX_DIRECT_CONF|NGX_CONF_TAKE1, + ngx_conf_set_str_slot, + 0, + offsetof(ngx_lua_resty_lmdb_conf_t, validation_tag), + NULL }, + ngx_null_command }; diff --git a/src/ngx_lua_resty_lmdb_module.h b/src/ngx_lua_resty_lmdb_module.h index 5a7a805d..9cd340ae 100644 --- a/src/ngx_lua_resty_lmdb_module.h +++ b/src/ngx_lua_resty_lmdb_module.h @@ -13,6 +13,8 @@ struct ngx_lua_resty_lmdb_conf_s { size_t map_size; MDB_env *env; MDB_txn *ro_txn; + + ngx_str_t validation_tag; }; From d64ab66f00239f1e970dd0c480b2a4fe74d3d7a4 Mon Sep 17 00:00:00 2001 From: chronolaw Date: Wed, 10 May 2023 14:48:57 +0800 Subject: [PATCH 27/70] ngx_lua_resty_lmdb_validate --- src/ngx_lua_resty_lmdb_module.c | 71 +++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/src/ngx_lua_resty_lmdb_module.c b/src/ngx_lua_resty_lmdb_module.c index 3aff321f..e7ac4378 100644 --- a/src/ngx_lua_resty_lmdb_module.c +++ b/src/ngx_lua_resty_lmdb_module.c @@ -5,6 +5,10 @@ #define NGX_LUA_RESTY_LMDB_DIR_MODE 0700 +#define NGX_LUA_RESTY_LMDB_VALIDATION_KEY "validation_tag" +#define NGX_LUA_RESTY_LMDB_DEFAULT_DB "_default" + + static ngx_str_t ngx_lua_resty_lmdb_file_names[] = { ngx_string("/data.mdb"), ngx_string("/lock.mdb"), @@ -370,6 +374,68 @@ ngx_lua_resty_lmdb_verify_file_status(ngx_cycle_t *cycle, } +static ngx_int_t +ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, + ngx_lua_resty_lmdb_conf_t *lcf) +{ + int rc; + MDB_dbi dbi; + MDB_val key; + MDB_val value; + MDB_txn *txn = lcf->ro_txn; + + ngx_str_t validation_key = + ngx_string(NGX_LUA_RESTY_LMDB_VALIDATION_KEY); + ngx_str_t default_db = + ngx_string(NGX_LUA_RESTY_LMDB_DEFAULT_DB); + + /* check tag value in lmdb */ + + if (lcf->validation_tag.data == NULL) { + return NGX_OK; + } + + rc = mdb_dbi_open(txn, (const char *) default_db.data, 0, &dbi); + if (rc != 0) { + ngx_log_error(NGX_LOG_ERR, cycle->log, 0, + "unable to open LMDB transaction : %s", + mdb_strerror(rc)); + return NGX_ERROR; + } + + key.mv_size = validation_key.len; + key.mv_data = validation_key.data; + + rc = mdb_get(txn, dbi, &key, &value); + if (rc == 0) { + /* key found, compare with validation_tag value */ + if (ngx_strncmp(lcf->validation_tag.data, + value.mv_data, value.mv_size) == 0) { + + return NGX_OK; + } + + ngx_log_error(NGX_LOG_WARN, cycle->log, 0, + "LMDB validation failed"); + + } else if (rc == MDB_NOTFOUND) { + ngx_log_error(NGX_LOG_WARN, cycle->log, 0, + "LMDB has no validation_tag"); + + } else { + ngx_log_error(NGX_LOG_ERR, cycle->log, 0, + "unable to get LMDB validation_tag : %s", + mdb_strerror(rc)); + mdb_txn_reset(txn); + return NGX_ERROR; + } + + /* drop lmdb db */ + + return NGX_OK; +} + + static ngx_int_t ngx_lua_resty_lmdb_init(ngx_cycle_t *cycle) { ngx_lua_resty_lmdb_conf_t *lcf; @@ -387,6 +453,11 @@ static ngx_int_t ngx_lua_resty_lmdb_init(ngx_cycle_t *cycle) return NGX_ERROR; } + if (ngx_lua_resty_lmdb_validate(cycle, lcf) != NGX_OK) { + ngx_lua_resty_lmdb_close_file(cycle, lcf); + return NGX_ERROR; + } + if (ngx_lua_resty_lmdb_close_file(cycle, lcf) != NGX_OK) { return NGX_ERROR; } From abc9d2fc824070d42da4d6aaf643d58af5b0672e Mon Sep 17 00:00:00 2001 From: chronolaw Date: Wed, 10 May 2023 15:09:53 +0800 Subject: [PATCH 28/70] ngx_lua_resty_lmdb_validate --- src/ngx_lua_resty_lmdb_module.c | 63 +++++++++++++++++++++++++++++++-- 1 file changed, 60 insertions(+), 3 deletions(-) diff --git a/src/ngx_lua_resty_lmdb_module.c b/src/ngx_lua_resty_lmdb_module.c index e7ac4378..bc77da09 100644 --- a/src/ngx_lua_resty_lmdb_module.c +++ b/src/ngx_lua_resty_lmdb_module.c @@ -395,10 +395,18 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, return NGX_OK; } + rc = mdb_txn_renew(txn); + if (rc != 0) { + ngx_log_error(NGX_LOG_ERR, cycle->log, 0, + "unable to renew LMDB transaction : %s", + mdb_strerror(rc)); + return NGX_ERROR; + } + rc = mdb_dbi_open(txn, (const char *) default_db.data, 0, &dbi); if (rc != 0) { ngx_log_error(NGX_LOG_ERR, cycle->log, 0, - "unable to open LMDB transaction : %s", + "unable to open LMDB database : %s", mdb_strerror(rc)); return NGX_ERROR; } @@ -412,6 +420,7 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, if (ngx_strncmp(lcf->validation_tag.data, value.mv_data, value.mv_size) == 0) { + mdb_txn_reset(txn); return NGX_OK; } @@ -426,13 +435,61 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, ngx_log_error(NGX_LOG_ERR, cycle->log, 0, "unable to get LMDB validation_tag : %s", mdb_strerror(rc)); - mdb_txn_reset(txn); - return NGX_ERROR; } + mdb_txn_reset(txn); + /* drop lmdb db */ + rc = mdb_txn_begin(lcf->env, NULL, 0, &txn); + if (rc != 0) { + ngx_log_error(NGX_LOG_CRIT, cycle->log, 0, + "unable to open LMDB write transaction: %s", + mdb_strerror(rc)); + return NGX_ERROR; + } + + rc = mdb_dbi_open(txn, (const char *) default_db.data, 0, &dbi); + if (rc != 0) { + ngx_log_error(NGX_LOG_ERR, cycle->log, 0, + "unable to open LMDB database : %s", + mdb_strerror(rc)); + return NGX_ERROR; + } + + rc = mdb_drop(txn, dbi, 0); + if (rc != 0) { + ngx_log_error(NGX_LOG_ERR, cycle->log, 0, + "unable to drop LMDB database : %s", + mdb_strerror(rc)); + goto failed; + } + + value.mv_size = lcf->validation_tag.len; + value.mv_data = lcf->validation_tag.data; + + rc = mdb_put(txn, dbi, &key, &value, 0); + if (rc != 0) { + ngx_log_error(NGX_LOG_ERR, cycle->log, 0, + "unable to set LMDB validation_tag : %s", + mdb_strerror(rc)); + goto failed; + } + + rc = mdb_txn_commit(txn); + if (rc != 0) { + ngx_log_error(NGX_LOG_ERR, cycle->log, 0, + "unable to commit LMDB : %s", + mdb_strerror(rc)); + goto failed; + } + return NGX_OK; + +failed: + + mdb_txn_abort(txn); + return NGX_ERROR; } From 27154eb9c3094428557145a0b0ed7c5b1d0f905b Mon Sep 17 00:00:00 2001 From: chronolaw Date: Wed, 10 May 2023 15:22:15 +0800 Subject: [PATCH 29/70] clean --- src/ngx_lua_resty_lmdb_module.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/ngx_lua_resty_lmdb_module.c b/src/ngx_lua_resty_lmdb_module.c index bc77da09..76049d02 100644 --- a/src/ngx_lua_resty_lmdb_module.c +++ b/src/ngx_lua_resty_lmdb_module.c @@ -386,8 +386,6 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, ngx_str_t validation_key = ngx_string(NGX_LUA_RESTY_LMDB_VALIDATION_KEY); - ngx_str_t default_db = - ngx_string(NGX_LUA_RESTY_LMDB_DEFAULT_DB); /* check tag value in lmdb */ @@ -403,11 +401,12 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, return NGX_ERROR; } - rc = mdb_dbi_open(txn, (const char *) default_db.data, 0, &dbi); + rc = mdb_dbi_open(txn, NGX_LUA_RESTY_LMDB_DEFAULT_DB, 0, &dbi); if (rc != 0) { ngx_log_error(NGX_LOG_ERR, cycle->log, 0, "unable to open LMDB database : %s", mdb_strerror(rc)); + mdb_txn_reset(txn); return NGX_ERROR; } @@ -449,7 +448,7 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, return NGX_ERROR; } - rc = mdb_dbi_open(txn, (const char *) default_db.data, 0, &dbi); + rc = mdb_dbi_open(txn, NGX_LUA_RESTY_LMDB_DEFAULT_DB, 0, &dbi); if (rc != 0) { ngx_log_error(NGX_LOG_ERR, cycle->log, 0, "unable to open LMDB database : %s", From e43364e69aaf3dfadd7803752f640eae61b10099 Mon Sep 17 00:00:00 2001 From: chronolaw Date: Wed, 10 May 2023 15:27:17 +0800 Subject: [PATCH 30/70] ngx_lua_resty_lmdb_assert --- src/ngx_lua_resty_lmdb_module.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/ngx_lua_resty_lmdb_module.c b/src/ngx_lua_resty_lmdb_module.c index 76049d02..babd54e6 100644 --- a/src/ngx_lua_resty_lmdb_module.c +++ b/src/ngx_lua_resty_lmdb_module.c @@ -393,6 +393,8 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, return NGX_OK; } + ngx_lua_resty_lmdb_assert(txn); + rc = mdb_txn_renew(txn); if (rc != 0) { ngx_log_error(NGX_LOG_ERR, cycle->log, 0, From 20e80769739562ddc16a099fbcf16b25b06cfd76 Mon Sep 17 00:00:00 2001 From: chronolaw Date: Wed, 10 May 2023 17:53:35 +0800 Subject: [PATCH 31/70] fix crash in cipher --- src/ngx_lua_resty_lmdb_module.c | 63 ++++++++++++++++++++++++++------- 1 file changed, 51 insertions(+), 12 deletions(-) diff --git a/src/ngx_lua_resty_lmdb_module.c b/src/ngx_lua_resty_lmdb_module.c index babd54e6..a39cae34 100644 --- a/src/ngx_lua_resty_lmdb_module.c +++ b/src/ngx_lua_resty_lmdb_module.c @@ -16,6 +16,9 @@ static ngx_str_t ngx_lua_resty_lmdb_file_names[] = { }; +static ngx_cycle_t *ngx_lua_resty_lmdb_init_cycle; + + static void *ngx_lua_resty_lmdb_create_conf(ngx_cycle_t *cycle); static char *ngx_lua_resty_lmdb_init_conf(ngx_cycle_t *cycle, void *conf); static ngx_int_t ngx_lua_resty_lmdb_init(ngx_cycle_t *cycle); @@ -382,7 +385,7 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, MDB_dbi dbi; MDB_val key; MDB_val value; - MDB_txn *txn = lcf->ro_txn; + MDB_txn *txn = NULL; ngx_str_t validation_key = ngx_string(NGX_LUA_RESTY_LMDB_VALIDATION_KEY); @@ -393,22 +396,27 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, return NGX_OK; } - ngx_lua_resty_lmdb_assert(txn); + /* save cycle in init phase for ngx_lua_resty_lmdb_cipher() */ + ngx_lua_resty_lmdb_init_cycle = cycle; - rc = mdb_txn_renew(txn); + rc = mdb_txn_begin(lcf->env, NULL, 0, &txn); if (rc != 0) { - ngx_log_error(NGX_LOG_ERR, cycle->log, 0, - "unable to renew LMDB transaction : %s", + ngx_log_error(NGX_LOG_CRIT, cycle->log, 0, + "unable to open LMDB transaction: %s", mdb_strerror(rc)); + ngx_lua_resty_lmdb_init_cycle = NULL; return NGX_ERROR; } - rc = mdb_dbi_open(txn, NGX_LUA_RESTY_LMDB_DEFAULT_DB, 0, &dbi); + ngx_lua_resty_lmdb_assert(txn); + + rc = mdb_dbi_open(txn, NGX_LUA_RESTY_LMDB_DEFAULT_DB, MDB_CREATE, &dbi); if (rc != 0) { ngx_log_error(NGX_LOG_ERR, cycle->log, 0, "unable to open LMDB database : %s", mdb_strerror(rc)); - mdb_txn_reset(txn); + ngx_lua_resty_lmdb_init_cycle = NULL; + mdb_txn_abort(txn); return NGX_ERROR; } @@ -421,7 +429,8 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, if (ngx_strncmp(lcf->validation_tag.data, value.mv_data, value.mv_size) == 0) { - mdb_txn_reset(txn); + ngx_lua_resty_lmdb_init_cycle = NULL; + mdb_txn_abort(txn); return NGX_OK; } @@ -438,7 +447,7 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, mdb_strerror(rc)); } - mdb_txn_reset(txn); + mdb_txn_abort(txn); /* drop lmdb db */ @@ -447,14 +456,16 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, ngx_log_error(NGX_LOG_CRIT, cycle->log, 0, "unable to open LMDB write transaction: %s", mdb_strerror(rc)); + ngx_lua_resty_lmdb_init_cycle = NULL; return NGX_ERROR; } - rc = mdb_dbi_open(txn, NGX_LUA_RESTY_LMDB_DEFAULT_DB, 0, &dbi); + rc = mdb_dbi_open(txn, NGX_LUA_RESTY_LMDB_DEFAULT_DB, MDB_CREATE, &dbi); if (rc != 0) { ngx_log_error(NGX_LOG_ERR, cycle->log, 0, "unable to open LMDB database : %s", mdb_strerror(rc)); + ngx_lua_resty_lmdb_init_cycle = NULL; return NGX_ERROR; } @@ -466,6 +477,33 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, goto failed; } + rc = mdb_txn_commit(txn); + if (rc != 0) { + ngx_log_error(NGX_LOG_ERR, cycle->log, 0, + "unable to commit LMDB : %s", + mdb_strerror(rc)); + goto failed; + } + + /* set tag to lmdb db */ + + rc = mdb_txn_begin(lcf->env, NULL, 0, &txn); + if (rc != 0) { + ngx_log_error(NGX_LOG_CRIT, cycle->log, 0, + "unable to open LMDB write transaction: %s", + mdb_strerror(rc)); + return NGX_ERROR; + } + + rc = mdb_dbi_open(txn, NGX_LUA_RESTY_LMDB_DEFAULT_DB, MDB_CREATE, &dbi); + if (rc != 0) { + ngx_log_error(NGX_LOG_ERR, cycle->log, 0, + "unable to open LMDB database : %s", + mdb_strerror(rc)); + ngx_lua_resty_lmdb_init_cycle = NULL; + return NGX_ERROR; + } + value.mv_size = lcf->validation_tag.len; value.mv_data = lcf->validation_tag.data; @@ -485,10 +523,13 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, goto failed; } + ngx_lua_resty_lmdb_init_cycle = NULL; + return NGX_OK; failed: + ngx_lua_resty_lmdb_init_cycle = NULL; mdb_txn_abort(txn); return NGX_ERROR; } @@ -564,5 +605,3 @@ static void ngx_lua_resty_lmdb_exit_worker(ngx_cycle_t *cycle) ngx_lua_resty_lmdb_close_file(cycle, lcf); } } - - From a643bda32a3795419829655f6b3f7f4d2f32839d Mon Sep 17 00:00:00 2001 From: chronolaw Date: Wed, 10 May 2023 18:02:49 +0800 Subject: [PATCH 32/70] ngx_lua_resty_lmdb_ngx_cycle --- src/ngx_lua_resty_lmdb_module.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/ngx_lua_resty_lmdb_module.c b/src/ngx_lua_resty_lmdb_module.c index a39cae34..17dbfcd2 100644 --- a/src/ngx_lua_resty_lmdb_module.c +++ b/src/ngx_lua_resty_lmdb_module.c @@ -16,7 +16,7 @@ static ngx_str_t ngx_lua_resty_lmdb_file_names[] = { }; -static ngx_cycle_t *ngx_lua_resty_lmdb_init_cycle; +static ngx_cycle_t *ngx_lua_resty_lmdb_ngx_cycle; static void *ngx_lua_resty_lmdb_create_conf(ngx_cycle_t *cycle); @@ -397,14 +397,14 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, } /* save cycle in init phase for ngx_lua_resty_lmdb_cipher() */ - ngx_lua_resty_lmdb_init_cycle = cycle; + ngx_lua_resty_lmdb_ngx_cycle = cycle; rc = mdb_txn_begin(lcf->env, NULL, 0, &txn); if (rc != 0) { ngx_log_error(NGX_LOG_CRIT, cycle->log, 0, "unable to open LMDB transaction: %s", mdb_strerror(rc)); - ngx_lua_resty_lmdb_init_cycle = NULL; + ngx_lua_resty_lmdb_ngx_cycle = NULL; return NGX_ERROR; } @@ -415,7 +415,7 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, ngx_log_error(NGX_LOG_ERR, cycle->log, 0, "unable to open LMDB database : %s", mdb_strerror(rc)); - ngx_lua_resty_lmdb_init_cycle = NULL; + ngx_lua_resty_lmdb_ngx_cycle = NULL; mdb_txn_abort(txn); return NGX_ERROR; } @@ -429,7 +429,7 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, if (ngx_strncmp(lcf->validation_tag.data, value.mv_data, value.mv_size) == 0) { - ngx_lua_resty_lmdb_init_cycle = NULL; + ngx_lua_resty_lmdb_ngx_cycle = NULL; mdb_txn_abort(txn); return NGX_OK; } @@ -456,7 +456,7 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, ngx_log_error(NGX_LOG_CRIT, cycle->log, 0, "unable to open LMDB write transaction: %s", mdb_strerror(rc)); - ngx_lua_resty_lmdb_init_cycle = NULL; + ngx_lua_resty_lmdb_ngx_cycle = NULL; return NGX_ERROR; } @@ -465,7 +465,7 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, ngx_log_error(NGX_LOG_ERR, cycle->log, 0, "unable to open LMDB database : %s", mdb_strerror(rc)); - ngx_lua_resty_lmdb_init_cycle = NULL; + ngx_lua_resty_lmdb_ngx_cycle = NULL; return NGX_ERROR; } @@ -500,7 +500,7 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, ngx_log_error(NGX_LOG_ERR, cycle->log, 0, "unable to open LMDB database : %s", mdb_strerror(rc)); - ngx_lua_resty_lmdb_init_cycle = NULL; + ngx_lua_resty_lmdb_ngx_cycle = NULL; return NGX_ERROR; } @@ -523,13 +523,13 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, goto failed; } - ngx_lua_resty_lmdb_init_cycle = NULL; + ngx_lua_resty_lmdb_ngx_cycle = NULL; return NGX_OK; failed: - ngx_lua_resty_lmdb_init_cycle = NULL; + ngx_lua_resty_lmdb_ngx_cycle = NULL; mdb_txn_abort(txn); return NGX_ERROR; } From 45a95653150e5b584b28dd514ab61403aadfda5a Mon Sep 17 00:00:00 2001 From: chronolaw Date: Wed, 10 May 2023 18:08:51 +0800 Subject: [PATCH 33/70] clear init_cycle --- src/ngx_lua_resty_lmdb_module.c | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/src/ngx_lua_resty_lmdb_module.c b/src/ngx_lua_resty_lmdb_module.c index 17dbfcd2..56fe52c0 100644 --- a/src/ngx_lua_resty_lmdb_module.c +++ b/src/ngx_lua_resty_lmdb_module.c @@ -396,15 +396,11 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, return NGX_OK; } - /* save cycle in init phase for ngx_lua_resty_lmdb_cipher() */ - ngx_lua_resty_lmdb_ngx_cycle = cycle; - rc = mdb_txn_begin(lcf->env, NULL, 0, &txn); if (rc != 0) { ngx_log_error(NGX_LOG_CRIT, cycle->log, 0, "unable to open LMDB transaction: %s", mdb_strerror(rc)); - ngx_lua_resty_lmdb_ngx_cycle = NULL; return NGX_ERROR; } @@ -415,7 +411,6 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, ngx_log_error(NGX_LOG_ERR, cycle->log, 0, "unable to open LMDB database : %s", mdb_strerror(rc)); - ngx_lua_resty_lmdb_ngx_cycle = NULL; mdb_txn_abort(txn); return NGX_ERROR; } @@ -429,7 +424,6 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, if (ngx_strncmp(lcf->validation_tag.data, value.mv_data, value.mv_size) == 0) { - ngx_lua_resty_lmdb_ngx_cycle = NULL; mdb_txn_abort(txn); return NGX_OK; } @@ -456,7 +450,6 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, ngx_log_error(NGX_LOG_CRIT, cycle->log, 0, "unable to open LMDB write transaction: %s", mdb_strerror(rc)); - ngx_lua_resty_lmdb_ngx_cycle = NULL; return NGX_ERROR; } @@ -465,7 +458,6 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, ngx_log_error(NGX_LOG_ERR, cycle->log, 0, "unable to open LMDB database : %s", mdb_strerror(rc)); - ngx_lua_resty_lmdb_ngx_cycle = NULL; return NGX_ERROR; } @@ -500,7 +492,6 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, ngx_log_error(NGX_LOG_ERR, cycle->log, 0, "unable to open LMDB database : %s", mdb_strerror(rc)); - ngx_lua_resty_lmdb_ngx_cycle = NULL; return NGX_ERROR; } @@ -523,13 +514,10 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, goto failed; } - ngx_lua_resty_lmdb_ngx_cycle = NULL; - return NGX_OK; failed: - ngx_lua_resty_lmdb_ngx_cycle = NULL; mdb_txn_abort(txn); return NGX_ERROR; } @@ -552,11 +540,17 @@ static ngx_int_t ngx_lua_resty_lmdb_init(ngx_cycle_t *cycle) return NGX_ERROR; } + /* save cycle in init phase for ngx_lua_resty_lmdb_cipher() */ + ngx_lua_resty_lmdb_ngx_cycle = cycle; + if (ngx_lua_resty_lmdb_validate(cycle, lcf) != NGX_OK) { ngx_lua_resty_lmdb_close_file(cycle, lcf); return NGX_ERROR; } + /* set to NULL fro worker processes */ + ngx_lua_resty_lmdb_ngx_cycle = NULL; + if (ngx_lua_resty_lmdb_close_file(cycle, lcf) != NGX_OK) { return NGX_ERROR; } From 7c60621c7eb4df63a605e2297c6ecfef912cb259 Mon Sep 17 00:00:00 2001 From: chronolaw Date: Wed, 10 May 2023 18:13:41 +0800 Subject: [PATCH 34/70] remove some redundant code --- src/ngx_lua_resty_lmdb_module.c | 26 +------------------------- 1 file changed, 1 insertion(+), 25 deletions(-) diff --git a/src/ngx_lua_resty_lmdb_module.c b/src/ngx_lua_resty_lmdb_module.c index 56fe52c0..6b1f85e8 100644 --- a/src/ngx_lua_resty_lmdb_module.c +++ b/src/ngx_lua_resty_lmdb_module.c @@ -469,31 +469,7 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, goto failed; } - rc = mdb_txn_commit(txn); - if (rc != 0) { - ngx_log_error(NGX_LOG_ERR, cycle->log, 0, - "unable to commit LMDB : %s", - mdb_strerror(rc)); - goto failed; - } - - /* set tag to lmdb db */ - - rc = mdb_txn_begin(lcf->env, NULL, 0, &txn); - if (rc != 0) { - ngx_log_error(NGX_LOG_CRIT, cycle->log, 0, - "unable to open LMDB write transaction: %s", - mdb_strerror(rc)); - return NGX_ERROR; - } - - rc = mdb_dbi_open(txn, NGX_LUA_RESTY_LMDB_DEFAULT_DB, MDB_CREATE, &dbi); - if (rc != 0) { - ngx_log_error(NGX_LOG_ERR, cycle->log, 0, - "unable to open LMDB database : %s", - mdb_strerror(rc)); - return NGX_ERROR; - } + /* set tag value to lmdb db */ value.mv_size = lcf->validation_tag.len; value.mv_data = lcf->validation_tag.data; From 03e21ddc5abe5e06006f6652a65bde34d5249a7e Mon Sep 17 00:00:00 2001 From: chronolaw Date: Thu, 11 May 2023 09:27:29 +0800 Subject: [PATCH 35/70] MDB_CRYPTO_FAIL --- src/ngx_lua_resty_lmdb_module.c | 28 ++++++++-------------------- 1 file changed, 8 insertions(+), 20 deletions(-) diff --git a/src/ngx_lua_resty_lmdb_module.c b/src/ngx_lua_resty_lmdb_module.c index 6b1f85e8..697fcacc 100644 --- a/src/ngx_lua_resty_lmdb_module.c +++ b/src/ngx_lua_resty_lmdb_module.c @@ -411,8 +411,13 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, ngx_log_error(NGX_LOG_ERR, cycle->log, 0, "unable to open LMDB database : %s", mdb_strerror(rc)); - mdb_txn_abort(txn); - return NGX_ERROR; + + if (rc == MDB_CRYPTO_FAIL) { + mdb_txn_abort(txn); + return NGX_DECLINED; + } + + goto failed; } key.mv_size = validation_key.len; @@ -439,28 +444,11 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, ngx_log_error(NGX_LOG_ERR, cycle->log, 0, "unable to get LMDB validation_tag : %s", mdb_strerror(rc)); + goto failed; } - mdb_txn_abort(txn); - /* drop lmdb db */ - rc = mdb_txn_begin(lcf->env, NULL, 0, &txn); - if (rc != 0) { - ngx_log_error(NGX_LOG_CRIT, cycle->log, 0, - "unable to open LMDB write transaction: %s", - mdb_strerror(rc)); - return NGX_ERROR; - } - - rc = mdb_dbi_open(txn, NGX_LUA_RESTY_LMDB_DEFAULT_DB, MDB_CREATE, &dbi); - if (rc != 0) { - ngx_log_error(NGX_LOG_ERR, cycle->log, 0, - "unable to open LMDB database : %s", - mdb_strerror(rc)); - return NGX_ERROR; - } - rc = mdb_drop(txn, dbi, 0); if (rc != 0) { ngx_log_error(NGX_LOG_ERR, cycle->log, 0, From 83af2aff3f1bcba5b5a05ca618b3b37627e2635b Mon Sep 17 00:00:00 2001 From: chronolaw Date: Thu, 11 May 2023 10:11:14 +0800 Subject: [PATCH 36/70] t/10-validation-tag.t --- t/10-validation-tag.t | 92 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 t/10-validation-tag.t diff --git a/t/10-validation-tag.t b/t/10-validation-tag.t new file mode 100644 index 00000000..83cbe88a --- /dev/null +++ b/t/10-validation-tag.t @@ -0,0 +1,92 @@ +# vim:set ft= ts=4 sw=4 et: + +# run this test after 00-init_mdb.t + +use Test::Nginx::Socket::Lua 'no_plan'; +use Cwd qw(cwd); + +repeat_each(1); + +my $pwd = cwd(); + +our $MainConfig1 = qq{ + lmdb_environment_path /tmp/test10.mdb; + lmdb_map_size 5m; +}; + +our $MainConfig2 = qq{ + lmdb_environment_path /tmp/test10.mdb; + lmdb_map_size 5m; + lmdb_validation_tag 3.3; +}; + +our $HttpConfig = qq{ + lua_package_path "$pwd/lib/?.lua;;"; +}; + +no_long_string(); +#no_diff(); + +no_shuffle(); +run_tests(); + +__DATA__ + +=== TEST 1: no validation_tag +--- http_config eval: $::HttpConfig +--- main_config eval: $::MainConfig1 +--- config + location = /t { + content_by_lua_block { + local l = require("resty.lmdb") + + ngx.say(l.set("test", "value")) + ngx.say(l.get("test")) + ngx.say(l.get("test_not_exist")) + + ngx.say(l.get("validation_tag")) + } + } +--- request +GET /t +--- response_body +true +value +nil +nil +--- no_error_log +[error] +[warn] +[crit] + + +=== TEST 2: start and set validation_tag +--- http_config eval: $::HttpConfig +--- main_config eval: $::MainConfig2 +--- config + location = /t { + content_by_lua_block { + local l = require("resty.lmdb") + + ngx.say(l.get("validation_tag")) + ngx.say(l.get("test")) + + ngx.say(l.set("test", "value")) + ngx.say(l.get("test")) + ngx.say(l.get("test_not_exist")) + } + } +--- request +GET /t +--- response_body +3.3 +nil +true +value +nil +--- error_log +LMDB has no validation_tag +--- no_error_log +[emerg] +[error] +[crit] From c4fce31bd9b040b069705e436506118cc58a3b97 Mon Sep 17 00:00:00 2001 From: chronolaw Date: Thu, 11 May 2023 10:22:03 +0800 Subject: [PATCH 37/70] 10-validation-tag-cipher.t --- t/10-validation-tag-cipher.t | 101 ++++++++++++++++++ ...dation-tag.t => 10-validation-tag-plain.t} | 11 +- 2 files changed, 109 insertions(+), 3 deletions(-) create mode 100644 t/10-validation-tag-cipher.t rename t/{10-validation-tag.t => 10-validation-tag-plain.t} (85%) diff --git a/t/10-validation-tag-cipher.t b/t/10-validation-tag-cipher.t new file mode 100644 index 00000000..9263c9a3 --- /dev/null +++ b/t/10-validation-tag-cipher.t @@ -0,0 +1,101 @@ +# vim:set ft= ts=4 sw=4 et: + +# run this test after 00-init_mdb.t + +use Test::Nginx::Socket::Lua; +use Cwd qw(cwd); + +repeat_each(1); + +plan tests => repeat_each() * blocks() * 5 + 1; + +my $pwd = cwd(); + +# remove db for testing +system("rm -rf /tmp/test10-cipher.mdb"); + +our $MainConfig1 = qq{ + lmdb_environment_path /tmp/test10-cipher.mdb; + lmdb_map_size 5m; + lmdb_encryption_key /etc/hostname; + lmdb_encryption_mode "chacha20-poly1305"; +}; + +our $MainConfig2 = qq{ + lmdb_environment_path /tmp/test10-cipher.mdb; + lmdb_map_size 5m; + lmdb_encryption_key /etc/hostname; + lmdb_encryption_mode "chacha20-poly1305"; + lmdb_validation_tag 3.3; +}; + +our $HttpConfig = qq{ + lua_package_path "$pwd/lib/?.lua;;"; +}; + +no_long_string(); +#no_diff(); + +no_shuffle(); +run_tests(); + +__DATA__ + +=== TEST 1: no validation_tag +--- http_config eval: $::HttpConfig +--- main_config eval: $::MainConfig1 +--- config + location = /t { + content_by_lua_block { + local l = require("resty.lmdb") + + ngx.say(l.set("test", "value")) + ngx.say(l.get("test")) + ngx.say(l.get("test_not_exist")) + + ngx.say(l.get("validation_tag")) + } + } +--- request +GET /t +--- response_body +true +value +nil +nil +--- no_error_log +[error] +[warn] +[crit] + + +=== TEST 2: start and set validation_tag +--- http_config eval: $::HttpConfig +--- main_config eval: $::MainConfig2 +--- config + location = /t { + content_by_lua_block { + local l = require("resty.lmdb") + + ngx.say(l.get("validation_tag")) + ngx.say(l.get("test")) + + ngx.say(l.set("test", "value")) + ngx.say(l.get("test")) + ngx.say(l.get("test_not_exist")) + } + } +--- request +GET /t +--- response_body +3.3 +nil +true +value +nil +--- error_log +LMDB has no validation_tag +--- no_error_log +[emerg] +[error] +[crit] diff --git a/t/10-validation-tag.t b/t/10-validation-tag-plain.t similarity index 85% rename from t/10-validation-tag.t rename to t/10-validation-tag-plain.t index 83cbe88a..72360005 100644 --- a/t/10-validation-tag.t +++ b/t/10-validation-tag-plain.t @@ -2,20 +2,25 @@ # run this test after 00-init_mdb.t -use Test::Nginx::Socket::Lua 'no_plan'; +use Test::Nginx::Socket::Lua; use Cwd qw(cwd); repeat_each(1); +plan tests => repeat_each() * blocks() * 5 + 1; + my $pwd = cwd(); +# remove db for testing +system("rm -rf /tmp/test10-plain.mdb"); + our $MainConfig1 = qq{ - lmdb_environment_path /tmp/test10.mdb; + lmdb_environment_path /tmp/test10-plain.mdb; lmdb_map_size 5m; }; our $MainConfig2 = qq{ - lmdb_environment_path /tmp/test10.mdb; + lmdb_environment_path /tmp/test10-plain.mdb; lmdb_map_size 5m; lmdb_validation_tag 3.3; }; From 5c70fa6e8c1606e6359e9d94d9fb41905ce78dc1 Mon Sep 17 00:00:00 2001 From: chronolaw Date: Thu, 11 May 2023 10:25:14 +0800 Subject: [PATCH 38/70] more tests --- t/10-validation-tag-cipher.t | 42 +++++++++++++++++++++++++++++++++++- t/10-validation-tag-plain.t | 40 +++++++++++++++++++++++++++++++++- 2 files changed, 80 insertions(+), 2 deletions(-) diff --git a/t/10-validation-tag-cipher.t b/t/10-validation-tag-cipher.t index 9263c9a3..d7c978a9 100644 --- a/t/10-validation-tag-cipher.t +++ b/t/10-validation-tag-cipher.t @@ -7,7 +7,7 @@ use Cwd qw(cwd); repeat_each(1); -plan tests => repeat_each() * blocks() * 5 + 1; +plan tests => repeat_each() * blocks() * 5 + 2; my $pwd = cwd(); @@ -29,6 +29,14 @@ our $MainConfig2 = qq{ lmdb_validation_tag 3.3; }; +our $MainConfig3 = qq{ + lmdb_environment_path /tmp/test10-cipher.mdb; + lmdb_map_size 5m; + lmdb_encryption_key /etc/hostname; + lmdb_encryption_mode "chacha20-poly1305"; + lmdb_validation_tag 3.4; +}; + our $HttpConfig = qq{ lua_package_path "$pwd/lib/?.lua;;"; }; @@ -99,3 +107,35 @@ LMDB has no validation_tag [emerg] [error] [crit] + + +=== TEST 3: change validation_tag +--- http_config eval: $::HttpConfig +--- main_config eval: $::MainConfig3 +--- config + location = /t { + content_by_lua_block { + local l = require("resty.lmdb") + + ngx.say(l.get("validation_tag")) + ngx.say(l.get("test")) + + ngx.say(l.set("test", "value")) + ngx.say(l.get("test")) + ngx.say(l.get("test_not_exist")) + } + } +--- request +GET /t +--- response_body +3.4 +nil +true +value +nil +--- error_log +LMDB validation failed +--- no_error_log +[emerg] +[error] +[crit] diff --git a/t/10-validation-tag-plain.t b/t/10-validation-tag-plain.t index 72360005..13f6be96 100644 --- a/t/10-validation-tag-plain.t +++ b/t/10-validation-tag-plain.t @@ -7,7 +7,7 @@ use Cwd qw(cwd); repeat_each(1); -plan tests => repeat_each() * blocks() * 5 + 1; +plan tests => repeat_each() * blocks() * 5 + 2; my $pwd = cwd(); @@ -25,6 +25,12 @@ our $MainConfig2 = qq{ lmdb_validation_tag 3.3; }; +our $MainConfig3 = qq{ + lmdb_environment_path /tmp/test10-plain.mdb; + lmdb_map_size 5m; + lmdb_validation_tag 3.4; +}; + our $HttpConfig = qq{ lua_package_path "$pwd/lib/?.lua;;"; }; @@ -95,3 +101,35 @@ LMDB has no validation_tag [emerg] [error] [crit] + + +=== TEST 3: change validation_tag +--- http_config eval: $::HttpConfig +--- main_config eval: $::MainConfig3 +--- config + location = /t { + content_by_lua_block { + local l = require("resty.lmdb") + + ngx.say(l.get("validation_tag")) + ngx.say(l.get("test")) + + ngx.say(l.set("test", "value")) + ngx.say(l.get("test")) + ngx.say(l.get("test_not_exist")) + } + } +--- request +GET /t +--- response_body +3.4 +nil +true +value +nil +--- error_log +LMDB validation failed +--- no_error_log +[emerg] +[error] +[crit] From 99a3ee73ebe9ad68d0a54208097c57042b3857c7 Mon Sep 17 00:00:00 2001 From: chronolaw Date: Thu, 11 May 2023 10:49:04 +0800 Subject: [PATCH 39/70] clean --- src/ngx_lua_resty_lmdb_module.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/ngx_lua_resty_lmdb_module.c b/src/ngx_lua_resty_lmdb_module.c index 697fcacc..08d8f41c 100644 --- a/src/ngx_lua_resty_lmdb_module.c +++ b/src/ngx_lua_resty_lmdb_module.c @@ -411,12 +411,6 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, ngx_log_error(NGX_LOG_ERR, cycle->log, 0, "unable to open LMDB database : %s", mdb_strerror(rc)); - - if (rc == MDB_CRYPTO_FAIL) { - mdb_txn_abort(txn); - return NGX_DECLINED; - } - goto failed; } From f05a58788a66531bdf24a1f22fef8dc8b0a60e3d Mon Sep 17 00:00:00 2001 From: chronolaw Date: Thu, 11 May 2023 15:00:32 +0800 Subject: [PATCH 40/70] typo fix --- src/ngx_lua_resty_lmdb_module.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ngx_lua_resty_lmdb_module.c b/src/ngx_lua_resty_lmdb_module.c index 08d8f41c..11a5485d 100644 --- a/src/ngx_lua_resty_lmdb_module.c +++ b/src/ngx_lua_resty_lmdb_module.c @@ -506,7 +506,7 @@ static ngx_int_t ngx_lua_resty_lmdb_init(ngx_cycle_t *cycle) return NGX_ERROR; } - /* set to NULL fro worker processes */ + /* reset to NULL for worker processes */ ngx_lua_resty_lmdb_ngx_cycle = NULL; if (ngx_lua_resty_lmdb_close_file(cycle, lcf) != NGX_OK) { From fdaed441362e6dd6b89a2268f95f86cbc1cf6e75 Mon Sep 17 00:00:00 2001 From: chronolaw Date: Mon, 22 May 2023 16:36:06 +0800 Subject: [PATCH 41/70] remove cipher related code --- src/ngx_lua_resty_lmdb_module.c | 9 -- t/10-validation-tag-cipher.t | 141 -------------------------------- 2 files changed, 150 deletions(-) delete mode 100644 t/10-validation-tag-cipher.t diff --git a/src/ngx_lua_resty_lmdb_module.c b/src/ngx_lua_resty_lmdb_module.c index 11a5485d..c1c14877 100644 --- a/src/ngx_lua_resty_lmdb_module.c +++ b/src/ngx_lua_resty_lmdb_module.c @@ -16,9 +16,6 @@ static ngx_str_t ngx_lua_resty_lmdb_file_names[] = { }; -static ngx_cycle_t *ngx_lua_resty_lmdb_ngx_cycle; - - static void *ngx_lua_resty_lmdb_create_conf(ngx_cycle_t *cycle); static char *ngx_lua_resty_lmdb_init_conf(ngx_cycle_t *cycle, void *conf); static ngx_int_t ngx_lua_resty_lmdb_init(ngx_cycle_t *cycle); @@ -498,17 +495,11 @@ static ngx_int_t ngx_lua_resty_lmdb_init(ngx_cycle_t *cycle) return NGX_ERROR; } - /* save cycle in init phase for ngx_lua_resty_lmdb_cipher() */ - ngx_lua_resty_lmdb_ngx_cycle = cycle; - if (ngx_lua_resty_lmdb_validate(cycle, lcf) != NGX_OK) { ngx_lua_resty_lmdb_close_file(cycle, lcf); return NGX_ERROR; } - /* reset to NULL for worker processes */ - ngx_lua_resty_lmdb_ngx_cycle = NULL; - if (ngx_lua_resty_lmdb_close_file(cycle, lcf) != NGX_OK) { return NGX_ERROR; } diff --git a/t/10-validation-tag-cipher.t b/t/10-validation-tag-cipher.t deleted file mode 100644 index d7c978a9..00000000 --- a/t/10-validation-tag-cipher.t +++ /dev/null @@ -1,141 +0,0 @@ -# vim:set ft= ts=4 sw=4 et: - -# run this test after 00-init_mdb.t - -use Test::Nginx::Socket::Lua; -use Cwd qw(cwd); - -repeat_each(1); - -plan tests => repeat_each() * blocks() * 5 + 2; - -my $pwd = cwd(); - -# remove db for testing -system("rm -rf /tmp/test10-cipher.mdb"); - -our $MainConfig1 = qq{ - lmdb_environment_path /tmp/test10-cipher.mdb; - lmdb_map_size 5m; - lmdb_encryption_key /etc/hostname; - lmdb_encryption_mode "chacha20-poly1305"; -}; - -our $MainConfig2 = qq{ - lmdb_environment_path /tmp/test10-cipher.mdb; - lmdb_map_size 5m; - lmdb_encryption_key /etc/hostname; - lmdb_encryption_mode "chacha20-poly1305"; - lmdb_validation_tag 3.3; -}; - -our $MainConfig3 = qq{ - lmdb_environment_path /tmp/test10-cipher.mdb; - lmdb_map_size 5m; - lmdb_encryption_key /etc/hostname; - lmdb_encryption_mode "chacha20-poly1305"; - lmdb_validation_tag 3.4; -}; - -our $HttpConfig = qq{ - lua_package_path "$pwd/lib/?.lua;;"; -}; - -no_long_string(); -#no_diff(); - -no_shuffle(); -run_tests(); - -__DATA__ - -=== TEST 1: no validation_tag ---- http_config eval: $::HttpConfig ---- main_config eval: $::MainConfig1 ---- config - location = /t { - content_by_lua_block { - local l = require("resty.lmdb") - - ngx.say(l.set("test", "value")) - ngx.say(l.get("test")) - ngx.say(l.get("test_not_exist")) - - ngx.say(l.get("validation_tag")) - } - } ---- request -GET /t ---- response_body -true -value -nil -nil ---- no_error_log -[error] -[warn] -[crit] - - -=== TEST 2: start and set validation_tag ---- http_config eval: $::HttpConfig ---- main_config eval: $::MainConfig2 ---- config - location = /t { - content_by_lua_block { - local l = require("resty.lmdb") - - ngx.say(l.get("validation_tag")) - ngx.say(l.get("test")) - - ngx.say(l.set("test", "value")) - ngx.say(l.get("test")) - ngx.say(l.get("test_not_exist")) - } - } ---- request -GET /t ---- response_body -3.3 -nil -true -value -nil ---- error_log -LMDB has no validation_tag ---- no_error_log -[emerg] -[error] -[crit] - - -=== TEST 3: change validation_tag ---- http_config eval: $::HttpConfig ---- main_config eval: $::MainConfig3 ---- config - location = /t { - content_by_lua_block { - local l = require("resty.lmdb") - - ngx.say(l.get("validation_tag")) - ngx.say(l.get("test")) - - ngx.say(l.set("test", "value")) - ngx.say(l.get("test")) - ngx.say(l.get("test_not_exist")) - } - } ---- request -GET /t ---- response_body -3.4 -nil -true -value -nil ---- error_log -LMDB validation failed ---- no_error_log -[emerg] -[error] -[crit] From 47a2f57a3112ed1ddb1e88177345d3d7a2f3c553 Mon Sep 17 00:00:00 2001 From: chronolaw Date: Tue, 23 May 2023 15:45:37 +0800 Subject: [PATCH 42/70] debug log --- src/ngx_lua_resty_lmdb_module.c | 2 ++ t/10-validation-tag-plain.t | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/ngx_lua_resty_lmdb_module.c b/src/ngx_lua_resty_lmdb_module.c index c1c14877..ee1dc1d4 100644 --- a/src/ngx_lua_resty_lmdb_module.c +++ b/src/ngx_lua_resty_lmdb_module.c @@ -390,6 +390,8 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, /* check tag value in lmdb */ if (lcf->validation_tag.data == NULL) { + ngx_log_debug0(NGX_LOG_DEBUG_CORE, cycle->log, 0, + "LMDB validation disabled"); return NGX_OK; } diff --git a/t/10-validation-tag-plain.t b/t/10-validation-tag-plain.t index 13f6be96..45a7b23b 100644 --- a/t/10-validation-tag-plain.t +++ b/t/10-validation-tag-plain.t @@ -7,7 +7,7 @@ use Cwd qw(cwd); repeat_each(1); -plan tests => repeat_each() * blocks() * 5 + 2; +plan tests => repeat_each() * blocks() * 6; my $pwd = cwd(); @@ -65,6 +65,8 @@ true value nil nil +--- error_log +LMDB validation disabled --- no_error_log [error] [warn] From 43bdcf913fd9a1e3e891421e51f3088929e7b32b Mon Sep 17 00:00:00 2001 From: chronolaw Date: Tue, 30 May 2023 16:38:51 +0800 Subject: [PATCH 43/70] add assert --- src/ngx_lua_resty_lmdb_module.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/ngx_lua_resty_lmdb_module.c b/src/ngx_lua_resty_lmdb_module.c index ee1dc1d4..e94fe5bc 100644 --- a/src/ngx_lua_resty_lmdb_module.c +++ b/src/ngx_lua_resty_lmdb_module.c @@ -395,6 +395,8 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, return NGX_OK; } + ngx_lua_resty_lmdb_assert(lcf->validation_tag.data); + rc = mdb_txn_begin(lcf->env, NULL, 0, &txn); if (rc != 0) { ngx_log_error(NGX_LOG_CRIT, cycle->log, 0, From 471e8d3521eadfdd24ef2e3f321c7fabe7551a65 Mon Sep 17 00:00:00 2001 From: chronolaw Date: Tue, 4 Jul 2023 09:20:12 +0800 Subject: [PATCH 44/70] ngx_lua_resty_lmdb_write_tag --- src/ngx_lua_resty_lmdb_module.c | 38 +++++++++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/src/ngx_lua_resty_lmdb_module.c b/src/ngx_lua_resty_lmdb_module.c index e94fe5bc..442c8511 100644 --- a/src/ngx_lua_resty_lmdb_module.c +++ b/src/ngx_lua_resty_lmdb_module.c @@ -439,21 +439,51 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, ngx_log_error(NGX_LOG_ERR, cycle->log, 0, "unable to get LMDB validation_tag : %s", mdb_strerror(rc)); - goto failed; } - /* drop lmdb db */ + mdb_txn_abort(txn); + return NGX_ERROR; +} + - rc = mdb_drop(txn, dbi, 0); +static ngx_int_t +ngx_lua_resty_lmdb_write_tag(ngx_cycle_t *cycle, + ngx_lua_resty_lmdb_conf_t *lcf) +{ + int rc; + MDB_dbi dbi; + MDB_val key; + MDB_val value; + MDB_txn *txn = NULL; + + ngx_str_t validation_key = + ngx_string(NGX_LUA_RESTY_LMDB_VALIDATION_KEY); + + ngx_lua_resty_lmdb_assert(lcf->validation_tag.data); + + rc = mdb_txn_begin(lcf->env, NULL, 0, &txn); + if (rc != 0) { + ngx_log_error(NGX_LOG_CRIT, cycle->log, 0, + "unable to open LMDB transaction: %s", + mdb_strerror(rc)); + return NGX_ERROR; + } + + ngx_lua_resty_lmdb_assert(txn); + + rc = mdb_dbi_open(txn, NGX_LUA_RESTY_LMDB_DEFAULT_DB, MDB_CREATE, &dbi); if (rc != 0) { ngx_log_error(NGX_LOG_ERR, cycle->log, 0, - "unable to drop LMDB database : %s", + "unable to open LMDB database : %s", mdb_strerror(rc)); goto failed; } /* set tag value to lmdb db */ + key.mv_size = validation_key.len; + key.mv_data = validation_key.data; + value.mv_size = lcf->validation_tag.len; value.mv_data = lcf->validation_tag.data; From 6cf82c25e19e7db060dc6b62b3fb6413418693eb Mon Sep 17 00:00:00 2001 From: chronolaw Date: Tue, 4 Jul 2023 09:34:55 +0800 Subject: [PATCH 45/70] remove if validation failed --- src/ngx_lua_resty_lmdb_module.c | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/src/ngx_lua_resty_lmdb_module.c b/src/ngx_lua_resty_lmdb_module.c index 442c8511..e140d37c 100644 --- a/src/ngx_lua_resty_lmdb_module.c +++ b/src/ngx_lua_resty_lmdb_module.c @@ -441,6 +441,8 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, mdb_strerror(rc)); } +failed: + mdb_txn_abort(txn); return NGX_ERROR; } @@ -529,9 +531,26 @@ static ngx_int_t ngx_lua_resty_lmdb_init(ngx_cycle_t *cycle) return NGX_ERROR; } + /* check lmdb validation tag */ + if (ngx_lua_resty_lmdb_validate(cycle, lcf) != NGX_OK) { ngx_lua_resty_lmdb_close_file(cycle, lcf); - return NGX_ERROR; + + /* remove lmdb files to clear data */ + if (ngx_lua_resty_lmdb_remove_files(cycle, lcf->env_path) != NGX_OK) { + return NGX_ERROR; + } + + /* open lmdb file again */ + if (ngx_lua_resty_lmdb_open_file(cycle, lcf, 1) != NGX_OK) { + return NGX_ERROR; + } + + /* write tag into lmdb */ + if (ngx_lua_resty_lmdb_write_tag(cycle, lcf) != NGX_OK) { + ngx_lua_resty_lmdb_close_file(cycle, lcf); + return NGX_ERROR; + } } if (ngx_lua_resty_lmdb_close_file(cycle, lcf) != NGX_OK) { From a648a51dc8135fd52767ecfb5dec1c257139d7c5 Mon Sep 17 00:00:00 2001 From: chronolaw Date: Fri, 14 Jul 2023 14:09:15 +0800 Subject: [PATCH 46/70] style lint --- src/ngx_lua_resty_lmdb_module.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/ngx_lua_resty_lmdb_module.c b/src/ngx_lua_resty_lmdb_module.c index e140d37c..3d56f20e 100644 --- a/src/ngx_lua_resty_lmdb_module.c +++ b/src/ngx_lua_resty_lmdb_module.c @@ -601,3 +601,5 @@ static void ngx_lua_resty_lmdb_exit_worker(ngx_cycle_t *cycle) ngx_lua_resty_lmdb_close_file(cycle, lcf); } } + + From c250d0c01feffdc8792c5863bd30b76642c08a30 Mon Sep 17 00:00:00 2001 From: chronolaw Date: Thu, 24 Aug 2023 11:19:50 +0800 Subject: [PATCH 47/70] change ngx_lua_resty_lmdb_remove_files --- src/ngx_lua_resty_lmdb_module.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/ngx_lua_resty_lmdb_module.c b/src/ngx_lua_resty_lmdb_module.c index 3d56f20e..947bed30 100644 --- a/src/ngx_lua_resty_lmdb_module.c +++ b/src/ngx_lua_resty_lmdb_module.c @@ -161,13 +161,14 @@ ngx_lua_resty_lmdb_create_env(ngx_cycle_t *cycle, static ngx_int_t -ngx_lua_resty_lmdb_remove_files(ngx_cycle_t *cycle, ngx_path_t *path) +ngx_lua_resty_lmdb_remove_files(ngx_cycle_t *cycle, ngx_lua_resty_lmdb_conf_t *lcf) { ngx_file_info_t fi; u_char name_buf[NGX_MAX_PATH]; ngx_str_t *names = ngx_lua_resty_lmdb_file_names; ngx_str_t *name; + ngx_path_t *path = lcf->env_path; ngx_log_error(NGX_LOG_WARN, cycle->log, 0, "LMDB database is corrupted or incompatible, removing"); @@ -244,7 +245,7 @@ ngx_lua_resty_lmdb_open_file(ngx_cycle_t *cycle, mdb_env_close(lcf->env); lcf->env = NULL; - if (ngx_lua_resty_lmdb_remove_files(cycle, lcf->env_path) != NGX_OK) { + if (ngx_lua_resty_lmdb_remove_files(cycle, lcf) != NGX_OK) { return NGX_ERROR; } @@ -537,7 +538,7 @@ static ngx_int_t ngx_lua_resty_lmdb_init(ngx_cycle_t *cycle) ngx_lua_resty_lmdb_close_file(cycle, lcf); /* remove lmdb files to clear data */ - if (ngx_lua_resty_lmdb_remove_files(cycle, lcf->env_path) != NGX_OK) { + if (ngx_lua_resty_lmdb_remove_files(cycle, lcf) != NGX_OK) { return NGX_ERROR; } From 39064f3159803fb50ec7fa880ddaa01e449fc79c Mon Sep 17 00:00:00 2001 From: chronolaw Date: Sun, 3 Sep 2023 17:15:34 +0800 Subject: [PATCH 48/70] compare value.mv_size --- src/ngx_lua_resty_lmdb_module.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ngx_lua_resty_lmdb_module.c b/src/ngx_lua_resty_lmdb_module.c index 947bed30..7796aa79 100644 --- a/src/ngx_lua_resty_lmdb_module.c +++ b/src/ngx_lua_resty_lmdb_module.c @@ -422,7 +422,8 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, rc = mdb_get(txn, dbi, &key, &value); if (rc == 0) { /* key found, compare with validation_tag value */ - if (ngx_strncmp(lcf->validation_tag.data, + if (lcf->validation_tag.len == value.mv_size && + ngx_strncmp(lcf->validation_tag.data, value.mv_data, value.mv_size) == 0) { mdb_txn_abort(txn); From 84fb15613fb88731d3da2b8de6e297a0bb3f37f7 Mon Sep 17 00:00:00 2001 From: chronolaw Date: Wed, 13 Sep 2023 14:00:45 +0800 Subject: [PATCH 49/70] readme --- README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/README.md b/README.md index c0e5f238..b40046fd 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,7 @@ Table of Contents * [lmdb_environment_path](#lmdb_environment_path) * [lmdb_max_databases](#lmdb_max_databases) * [lmdb_map_size](#lmdb_map_size) + * [lmdb_validation_tag](#lmdb_validation_tag) * [Copyright and license](#copyright-and-license) ## APIs @@ -228,6 +229,19 @@ Set the size of the memory map, the default value is `1048576`(1MB). [Back to TOC](#table-of-contents) +### lmdb_validation_tag + +**syntax:** *lmdb_validation_tag value;* + +**context:** *main* + +Set a content validation tag into LMDB. +When LMDB starts, it will check the tag value, +if the value is different from the directive value, +the content of LMDB will be cleaned up. + +[Back to TOC](#table-of-contents) + ## Copyright and license Copyright (c) 2021-2022 Kong, Inc. From 5426fca4c8cf706601c9c882cebdec024ef6c518 Mon Sep 17 00:00:00 2001 From: chronolaw Date: Wed, 13 Sep 2023 14:04:20 +0800 Subject: [PATCH 50/70] typo fix --- src/ngx_lua_resty_lmdb_module.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ngx_lua_resty_lmdb_module.c b/src/ngx_lua_resty_lmdb_module.c index 7796aa79..02c93ef7 100644 --- a/src/ngx_lua_resty_lmdb_module.c +++ b/src/ngx_lua_resty_lmdb_module.c @@ -538,7 +538,7 @@ static ngx_int_t ngx_lua_resty_lmdb_init(ngx_cycle_t *cycle) if (ngx_lua_resty_lmdb_validate(cycle, lcf) != NGX_OK) { ngx_lua_resty_lmdb_close_file(cycle, lcf); - /* remove lmdb files to clear data */ + /* remove lmdb files to clean data */ if (ngx_lua_resty_lmdb_remove_files(cycle, lcf) != NGX_OK) { return NGX_ERROR; } From f934bab2c36aac014be46cc00fa057fba2b2a3e2 Mon Sep 17 00:00:00 2001 From: chronolaw Date: Mon, 13 Nov 2023 14:34:34 +0800 Subject: [PATCH 51/70] print validation_tag when enabled --- src/ngx_lua_resty_lmdb_module.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/ngx_lua_resty_lmdb_module.c b/src/ngx_lua_resty_lmdb_module.c index 02c93ef7..61f34309 100644 --- a/src/ngx_lua_resty_lmdb_module.c +++ b/src/ngx_lua_resty_lmdb_module.c @@ -391,11 +391,13 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, /* check tag value in lmdb */ if (lcf->validation_tag.data == NULL) { - ngx_log_debug0(NGX_LOG_DEBUG_CORE, cycle->log, 0, - "LMDB validation disabled"); return NGX_OK; } + ngx_log_debug0(NGX_LOG_DEBUG_CORE, cycle->log, 0, + "LMDB validation enabled, using validation tag: \"%V\"", + &lcf->validation_tag); + ngx_lua_resty_lmdb_assert(lcf->validation_tag.data); rc = mdb_txn_begin(lcf->env, NULL, 0, &txn); From 0116474bd6f76b0d1759c30f9fd786c2f8720793 Mon Sep 17 00:00:00 2001 From: chronolaw Date: Mon, 13 Nov 2023 14:44:11 +0800 Subject: [PATCH 52/70] change log messages --- src/ngx_lua_resty_lmdb_module.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/ngx_lua_resty_lmdb_module.c b/src/ngx_lua_resty_lmdb_module.c index 61f34309..af079e9b 100644 --- a/src/ngx_lua_resty_lmdb_module.c +++ b/src/ngx_lua_resty_lmdb_module.c @@ -394,7 +394,7 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, return NGX_OK; } - ngx_log_debug0(NGX_LOG_DEBUG_CORE, cycle->log, 0, + ngx_log_debug1(NGX_LOG_DEBUG_CORE, cycle->log, 0, "LMDB validation enabled, using validation tag: \"%V\"", &lcf->validation_tag); @@ -413,7 +413,7 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, rc = mdb_dbi_open(txn, NGX_LUA_RESTY_LMDB_DEFAULT_DB, MDB_CREATE, &dbi); if (rc != 0) { ngx_log_error(NGX_LOG_ERR, cycle->log, 0, - "unable to open LMDB database : %s", + "unable to open LMDB database: %s", mdb_strerror(rc)); goto failed; } @@ -433,11 +433,13 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, } ngx_log_error(NGX_LOG_WARN, cycle->log, 0, - "LMDB validation failed"); + "LMDB tag \"%*s\" did not match configured tag \"%V\"", + value.mv_size, value.mv_data, + &lcf->validation_tag); } else if (rc == MDB_NOTFOUND) { ngx_log_error(NGX_LOG_WARN, cycle->log, 0, - "LMDB has no validation_tag"); + "LMDB validation tag does not exist"); } else { ngx_log_error(NGX_LOG_ERR, cycle->log, 0, @@ -538,6 +540,9 @@ static ngx_int_t ngx_lua_resty_lmdb_init(ngx_cycle_t *cycle) /* check lmdb validation tag */ if (ngx_lua_resty_lmdb_validate(cycle, lcf) != NGX_OK) { + ngx_log_error(NGX_LOG_WARN, cycle->log, 0, + "LMDB database is removed for tag validation failure"); + ngx_lua_resty_lmdb_close_file(cycle, lcf); /* remove lmdb files to clean data */ From 9dfddbfb143998a2ed8f1869aca210c2edc46a02 Mon Sep 17 00:00:00 2001 From: chronolaw Date: Mon, 13 Nov 2023 15:36:54 +0800 Subject: [PATCH 53/70] fix tests --- src/ngx_lua_resty_lmdb_module.c | 12 ++++++------ t/10-validation-tag-plain.t | 13 ++++++++----- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/src/ngx_lua_resty_lmdb_module.c b/src/ngx_lua_resty_lmdb_module.c index af079e9b..e2a69f0f 100644 --- a/src/ngx_lua_resty_lmdb_module.c +++ b/src/ngx_lua_resty_lmdb_module.c @@ -170,9 +170,6 @@ ngx_lua_resty_lmdb_remove_files(ngx_cycle_t *cycle, ngx_lua_resty_lmdb_conf_t *l ngx_str_t *name; ngx_path_t *path = lcf->env_path; - ngx_log_error(NGX_LOG_WARN, cycle->log, 0, - "LMDB database is corrupted or incompatible, removing"); - if (ngx_file_info(path->name.data, &fi) == NGX_FILE_ERROR) { ngx_log_error(NGX_LOG_CRIT, cycle->log, ngx_errno, ngx_file_info_n " \"%s\" failed", path->name.data); @@ -245,6 +242,9 @@ ngx_lua_resty_lmdb_open_file(ngx_cycle_t *cycle, mdb_env_close(lcf->env); lcf->env = NULL; + ngx_log_error(NGX_LOG_WARN, cycle->log, 0, + "LMDB database is corrupted or incompatible, removing"); + if (ngx_lua_resty_lmdb_remove_files(cycle, lcf) != NGX_OK) { return NGX_ERROR; } @@ -540,11 +540,11 @@ static ngx_int_t ngx_lua_resty_lmdb_init(ngx_cycle_t *cycle) /* check lmdb validation tag */ if (ngx_lua_resty_lmdb_validate(cycle, lcf) != NGX_OK) { - ngx_log_error(NGX_LOG_WARN, cycle->log, 0, - "LMDB database is removed for tag validation failure"); - ngx_lua_resty_lmdb_close_file(cycle, lcf); + ngx_log_error(NGX_LOG_WARN, cycle->log, 0, + "LMDB database tag validation fails, removing"); + /* remove lmdb files to clean data */ if (ngx_lua_resty_lmdb_remove_files(cycle, lcf) != NGX_OK) { return NGX_ERROR; diff --git a/t/10-validation-tag-plain.t b/t/10-validation-tag-plain.t index 45a7b23b..300dda4f 100644 --- a/t/10-validation-tag-plain.t +++ b/t/10-validation-tag-plain.t @@ -7,7 +7,7 @@ use Cwd qw(cwd); repeat_each(1); -plan tests => repeat_each() * blocks() * 6; +plan tests => repeat_each() * blocks() * 7 + 1; my $pwd = cwd(); @@ -65,9 +65,8 @@ true value nil nil ---- error_log -LMDB validation disabled --- no_error_log +[warn] [error] [warn] [crit] @@ -98,7 +97,9 @@ true value nil --- error_log -LMDB has no validation_tag +LMDB validation enabled, using validation tag: "3.3" +LMDB validation tag does not exist +LMDB database tag validation fails, removing --- no_error_log [emerg] [error] @@ -130,7 +131,9 @@ true value nil --- error_log -LMDB validation failed +LMDB validation enabled, using validation tag: "3.4" +LMDB tag "3.3" did not match configured tag "3.4" +LMDB database tag validation fails, removing --- no_error_log [emerg] [error] From 56e68b3cd079cbc2d3606d6cc63635c983846e8c Mon Sep 17 00:00:00 2001 From: chronolaw Date: Mon, 13 Nov 2023 15:41:53 +0800 Subject: [PATCH 54/70] tests --- t/10-validation-tag-plain.t | 1 + 1 file changed, 1 insertion(+) diff --git a/t/10-validation-tag-plain.t b/t/10-validation-tag-plain.t index e70067ec..300dda4f 100644 --- a/t/10-validation-tag-plain.t +++ b/t/10-validation-tag-plain.t @@ -66,6 +66,7 @@ value nil nil --- no_error_log +[warn] [error] [warn] [crit] From 47f6ff356e93490abf9a5e8ed39e510ea7946009 Mon Sep 17 00:00:00 2001 From: chronolaw Date: Mon, 13 Nov 2023 17:38:07 +0800 Subject: [PATCH 55/70] clean temp var --- src/ngx_lua_resty_lmdb_module.c | 9 +++------ t/10-validation-tag-plain.t | 5 +++-- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/ngx_lua_resty_lmdb_module.c b/src/ngx_lua_resty_lmdb_module.c index e2a69f0f..ad9ec2c2 100644 --- a/src/ngx_lua_resty_lmdb_module.c +++ b/src/ngx_lua_resty_lmdb_module.c @@ -385,9 +385,6 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, MDB_val value; MDB_txn *txn = NULL; - ngx_str_t validation_key = - ngx_string(NGX_LUA_RESTY_LMDB_VALIDATION_KEY); - /* check tag value in lmdb */ if (lcf->validation_tag.data == NULL) { @@ -418,8 +415,8 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, goto failed; } - key.mv_size = validation_key.len; - key.mv_data = validation_key.data; + key.mv_size = sizeof(NGX_LUA_RESTY_LMDB_VALIDATION_KEY); + key.mv_data = NGX_LUA_RESTY_LMDB_VALIDATION_KEY; rc = mdb_get(txn, dbi, &key, &value); if (rc == 0) { @@ -543,7 +540,7 @@ static ngx_int_t ngx_lua_resty_lmdb_init(ngx_cycle_t *cycle) ngx_lua_resty_lmdb_close_file(cycle, lcf); ngx_log_error(NGX_LOG_WARN, cycle->log, 0, - "LMDB database tag validation fails, removing"); + "LMDB database tag mismatch, wiping the database"); /* remove lmdb files to clean data */ if (ngx_lua_resty_lmdb_remove_files(cycle, lcf) != NGX_OK) { diff --git a/t/10-validation-tag-plain.t b/t/10-validation-tag-plain.t index 300dda4f..3a13684f 100644 --- a/t/10-validation-tag-plain.t +++ b/t/10-validation-tag-plain.t @@ -6,6 +6,7 @@ use Test::Nginx::Socket::Lua; use Cwd qw(cwd); repeat_each(1); +no_shuffle(); plan tests => repeat_each() * blocks() * 7 + 1; @@ -99,7 +100,7 @@ nil --- error_log LMDB validation enabled, using validation tag: "3.3" LMDB validation tag does not exist -LMDB database tag validation fails, removing +LMDB database tag mismatch, wiping the database --- no_error_log [emerg] [error] @@ -133,7 +134,7 @@ nil --- error_log LMDB validation enabled, using validation tag: "3.4" LMDB tag "3.3" did not match configured tag "3.4" -LMDB database tag validation fails, removing +LMDB database tag mismatch, wiping the database --- no_error_log [emerg] [error] From ad292296f86d094edc425ecf03e658b4f2108223 Mon Sep 17 00:00:00 2001 From: chronolaw Date: Mon, 13 Nov 2023 17:54:27 +0800 Subject: [PATCH 56/70] key string --- src/ngx_lua_resty_lmdb_module.c | 16 ++++++++++------ t/10-validation-tag-plain.t | 5 +++-- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/ngx_lua_resty_lmdb_module.c b/src/ngx_lua_resty_lmdb_module.c index ad9ec2c2..e9469495 100644 --- a/src/ngx_lua_resty_lmdb_module.c +++ b/src/ngx_lua_resty_lmdb_module.c @@ -415,7 +415,7 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, goto failed; } - key.mv_size = sizeof(NGX_LUA_RESTY_LMDB_VALIDATION_KEY); + key.mv_size = sizeof(NGX_LUA_RESTY_LMDB_VALIDATION_KEY) - 1; key.mv_data = NGX_LUA_RESTY_LMDB_VALIDATION_KEY; rc = mdb_get(txn, dbi, &key, &value); @@ -430,7 +430,7 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, } ngx_log_error(NGX_LOG_WARN, cycle->log, 0, - "LMDB tag \"%*s\" did not match configured tag \"%V\"", + "LMDB validation tag \"%*s\" did not match configured tag \"%V\"", value.mv_size, value.mv_data, &lcf->validation_tag); @@ -440,7 +440,7 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, } else { ngx_log_error(NGX_LOG_ERR, cycle->log, 0, - "unable to get LMDB validation_tag : %s", + "unable to get LMDB validation tag: %s", mdb_strerror(rc)); } @@ -479,7 +479,7 @@ ngx_lua_resty_lmdb_write_tag(ngx_cycle_t *cycle, rc = mdb_dbi_open(txn, NGX_LUA_RESTY_LMDB_DEFAULT_DB, MDB_CREATE, &dbi); if (rc != 0) { ngx_log_error(NGX_LOG_ERR, cycle->log, 0, - "unable to open LMDB database : %s", + "unable to open LMDB database: %s", mdb_strerror(rc)); goto failed; } @@ -495,7 +495,7 @@ ngx_lua_resty_lmdb_write_tag(ngx_cycle_t *cycle, rc = mdb_put(txn, dbi, &key, &value, 0); if (rc != 0) { ngx_log_error(NGX_LOG_ERR, cycle->log, 0, - "unable to set LMDB validation_tag : %s", + "unable to set LMDB validation tag: %s", mdb_strerror(rc)); goto failed; } @@ -503,11 +503,15 @@ ngx_lua_resty_lmdb_write_tag(ngx_cycle_t *cycle, rc = mdb_txn_commit(txn); if (rc != 0) { ngx_log_error(NGX_LOG_ERR, cycle->log, 0, - "unable to commit LMDB : %s", + "unable to commit LMDB: %s", mdb_strerror(rc)); goto failed; } + ngx_log_debug1(NGX_LOG_DEBUG_CORE, cycle->log, 0, + "set LMDB validation tag: \"%V\"", + &lcf->validation_tag); + return NGX_OK; failed: diff --git a/t/10-validation-tag-plain.t b/t/10-validation-tag-plain.t index 3a13684f..53fc413c 100644 --- a/t/10-validation-tag-plain.t +++ b/t/10-validation-tag-plain.t @@ -8,7 +8,7 @@ use Cwd qw(cwd); repeat_each(1); no_shuffle(); -plan tests => repeat_each() * blocks() * 7 + 1; +plan tests => repeat_each() * blocks() * 7 + 2; my $pwd = cwd(); @@ -101,6 +101,7 @@ nil LMDB validation enabled, using validation tag: "3.3" LMDB validation tag does not exist LMDB database tag mismatch, wiping the database +set LMDB validation tag: "3.3" --- no_error_log [emerg] [error] @@ -133,7 +134,7 @@ value nil --- error_log LMDB validation enabled, using validation tag: "3.4" -LMDB tag "3.3" did not match configured tag "3.4" +LMDB validation tag "3.3" did not match configured tag "3.4" LMDB database tag mismatch, wiping the database --- no_error_log [emerg] From d34470bf8fc82f7b07ba3076a4d7c7f13b2775da Mon Sep 17 00:00:00 2001 From: chronolaw Date: Mon, 13 Nov 2023 18:08:59 +0800 Subject: [PATCH 57/70] unnamed db --- src/ngx_lua_resty_lmdb_module.c | 6 +++--- t/10-validation-tag-plain.t | 8 ++------ 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/src/ngx_lua_resty_lmdb_module.c b/src/ngx_lua_resty_lmdb_module.c index e9469495..6be62f3c 100644 --- a/src/ngx_lua_resty_lmdb_module.c +++ b/src/ngx_lua_resty_lmdb_module.c @@ -6,7 +6,7 @@ #define NGX_LUA_RESTY_LMDB_VALIDATION_KEY "validation_tag" -#define NGX_LUA_RESTY_LMDB_DEFAULT_DB "_default" +#define NGX_LUA_RESTY_LMDB_UNNAMED_DB NULL static ngx_str_t ngx_lua_resty_lmdb_file_names[] = { @@ -407,7 +407,7 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, ngx_lua_resty_lmdb_assert(txn); - rc = mdb_dbi_open(txn, NGX_LUA_RESTY_LMDB_DEFAULT_DB, MDB_CREATE, &dbi); + rc = mdb_dbi_open(txn, NGX_LUA_RESTY_LMDB_UNNAMED_DB, MDB_CREATE, &dbi); if (rc != 0) { ngx_log_error(NGX_LOG_ERR, cycle->log, 0, "unable to open LMDB database: %s", @@ -476,7 +476,7 @@ ngx_lua_resty_lmdb_write_tag(ngx_cycle_t *cycle, ngx_lua_resty_lmdb_assert(txn); - rc = mdb_dbi_open(txn, NGX_LUA_RESTY_LMDB_DEFAULT_DB, MDB_CREATE, &dbi); + rc = mdb_dbi_open(txn, NGX_LUA_RESTY_LMDB_UNNAMED_DB, MDB_CREATE, &dbi); if (rc != 0) { ngx_log_error(NGX_LOG_ERR, cycle->log, 0, "unable to open LMDB database: %s", diff --git a/t/10-validation-tag-plain.t b/t/10-validation-tag-plain.t index 53fc413c..71e8a757 100644 --- a/t/10-validation-tag-plain.t +++ b/t/10-validation-tag-plain.t @@ -81,7 +81,6 @@ nil content_by_lua_block { local l = require("resty.lmdb") - ngx.say(l.get("validation_tag")) ngx.say(l.get("test")) ngx.say(l.set("test", "value")) @@ -92,8 +91,7 @@ nil --- request GET /t --- response_body -3.3 -nil +nilunable to open DB for access: MDB_NOTFOUND: No matching key/data pair found true value nil @@ -116,7 +114,6 @@ set LMDB validation tag: "3.3" content_by_lua_block { local l = require("resty.lmdb") - ngx.say(l.get("validation_tag")) ngx.say(l.get("test")) ngx.say(l.set("test", "value")) @@ -127,8 +124,7 @@ set LMDB validation tag: "3.3" --- request GET /t --- response_body -3.4 -nil +nilunable to open DB for access: MDB_NOTFOUND: No matching key/data pair found true value nil From 55f45955e65d3838898b24f97cd0c6ef09285412 Mon Sep 17 00:00:00 2001 From: chronolaw Date: Mon, 13 Nov 2023 18:12:51 +0800 Subject: [PATCH 58/70] remove validation_key --- src/ngx_lua_resty_lmdb_module.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/ngx_lua_resty_lmdb_module.c b/src/ngx_lua_resty_lmdb_module.c index 6be62f3c..b9a6d260 100644 --- a/src/ngx_lua_resty_lmdb_module.c +++ b/src/ngx_lua_resty_lmdb_module.c @@ -461,9 +461,6 @@ ngx_lua_resty_lmdb_write_tag(ngx_cycle_t *cycle, MDB_val value; MDB_txn *txn = NULL; - ngx_str_t validation_key = - ngx_string(NGX_LUA_RESTY_LMDB_VALIDATION_KEY); - ngx_lua_resty_lmdb_assert(lcf->validation_tag.data); rc = mdb_txn_begin(lcf->env, NULL, 0, &txn); @@ -486,8 +483,8 @@ ngx_lua_resty_lmdb_write_tag(ngx_cycle_t *cycle, /* set tag value to lmdb db */ - key.mv_size = validation_key.len; - key.mv_data = validation_key.data; + key.mv_size = sizeof(NGX_LUA_RESTY_LMDB_VALIDATION_KEY) - 1; + key.mv_data = NGX_LUA_RESTY_LMDB_VALIDATION_KEY; value.mv_size = lcf->validation_tag.len; value.mv_data = lcf->validation_tag.data; From b3288c4f848b9c07877030c6e881c977c5cd1ea2 Mon Sep 17 00:00:00 2001 From: chronolaw Date: Mon, 13 Nov 2023 18:14:33 +0800 Subject: [PATCH 59/70] test clean --- t/10-validation-tag-plain.t | 1 - 1 file changed, 1 deletion(-) diff --git a/t/10-validation-tag-plain.t b/t/10-validation-tag-plain.t index 71e8a757..08cbd4b2 100644 --- a/t/10-validation-tag-plain.t +++ b/t/10-validation-tag-plain.t @@ -6,7 +6,6 @@ use Test::Nginx::Socket::Lua; use Cwd qw(cwd); repeat_each(1); -no_shuffle(); plan tests => repeat_each() * blocks() * 7 + 2; From 38cf3a6c1c722fa4a9b66057402d4eed1a7fb748 Mon Sep 17 00:00:00 2001 From: chronolaw Date: Mon, 13 Nov 2023 18:27:49 +0800 Subject: [PATCH 60/70] readme --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index b40046fd..5cf23f6c 100644 --- a/README.md +++ b/README.md @@ -240,6 +240,9 @@ When LMDB starts, it will check the tag value, if the value is different from the directive value, the content of LMDB will be cleaned up. +The default value is null string, +and this feature will be disabled. + [Back to TOC](#table-of-contents) ## Copyright and license From 11f37138dcf2f5cf41e3e50ab29d63eb20d78b68 Mon Sep 17 00:00:00 2001 From: chronolaw Date: Mon, 13 Nov 2023 19:00:22 +0800 Subject: [PATCH 61/70] debug log --- src/ngx_lua_resty_lmdb_module.c | 2 ++ t/10-validation-tag-plain.t | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/ngx_lua_resty_lmdb_module.c b/src/ngx_lua_resty_lmdb_module.c index b9a6d260..f862b8c8 100644 --- a/src/ngx_lua_resty_lmdb_module.c +++ b/src/ngx_lua_resty_lmdb_module.c @@ -388,6 +388,8 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, /* check tag value in lmdb */ if (lcf->validation_tag.data == NULL) { + ngx_log_debug0(NGX_LOG_DEBUG_CORE, cycle->log, 0, + "LMDB validation disabled"); return NGX_OK; } diff --git a/t/10-validation-tag-plain.t b/t/10-validation-tag-plain.t index 08cbd4b2..27753590 100644 --- a/t/10-validation-tag-plain.t +++ b/t/10-validation-tag-plain.t @@ -65,8 +65,9 @@ true value nil nil +--- error_log +LMDB validation disabled --- no_error_log -[warn] [error] [warn] [crit] From 46a64bc593039ae5a4fe02aec058747b6ca42c7a Mon Sep 17 00:00:00 2001 From: chronolaw Date: Mon, 13 Nov 2023 19:27:01 +0800 Subject: [PATCH 62/70] tests --- t/10-validation-tag-plain.t | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/t/10-validation-tag-plain.t b/t/10-validation-tag-plain.t index 27753590..b780e951 100644 --- a/t/10-validation-tag-plain.t +++ b/t/10-validation-tag-plain.t @@ -7,7 +7,7 @@ use Cwd qw(cwd); repeat_each(1); -plan tests => repeat_each() * blocks() * 7 + 2; +plan tests => repeat_each() * blocks() * 7 + 3; my $pwd = cwd(); @@ -68,8 +68,8 @@ nil --- error_log LMDB validation disabled --- no_error_log -[error] [warn] +[error] [crit] @@ -132,6 +132,7 @@ nil LMDB validation enabled, using validation tag: "3.4" LMDB validation tag "3.3" did not match configured tag "3.4" LMDB database tag mismatch, wiping the database +set LMDB validation tag: "3.4" --- no_error_log [emerg] [error] From fc3f90cfa1e68c69b0ca085f2f501ce0aadc969c Mon Sep 17 00:00:00 2001 From: chronolaw Date: Mon, 13 Nov 2023 20:27:01 +0800 Subject: [PATCH 63/70] README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5cf23f6c..72d5822f 100644 --- a/README.md +++ b/README.md @@ -241,7 +241,7 @@ if the value is different from the directive value, the content of LMDB will be cleaned up. The default value is null string, -and this feature will be disabled. +which means this feature is disabled by default. [Back to TOC](#table-of-contents) From c5308d4260963c7b058b8c2cc0ce78481467706c Mon Sep 17 00:00:00 2001 From: chronolaw Date: Tue, 14 Nov 2023 17:29:52 +0800 Subject: [PATCH 64/70] readme --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 72d5822f..5a6fc08c 100644 --- a/README.md +++ b/README.md @@ -233,6 +233,8 @@ Set the size of the memory map, the default value is `1048576`(1MB). **syntax:** *lmdb_validation_tag value;* +**default:** *none* + **context:** *main* Set a content validation tag into LMDB. @@ -240,8 +242,7 @@ When LMDB starts, it will check the tag value, if the value is different from the directive value, the content of LMDB will be cleaned up. -The default value is null string, -which means this feature is disabled by default. +When this directive is not set, tag is not validated. [Back to TOC](#table-of-contents) From 33a56a96b4e777d3d9690b8c873c9aceec45d9c4 Mon Sep 17 00:00:00 2001 From: chronolaw Date: Tue, 14 Nov 2023 17:31:19 +0800 Subject: [PATCH 65/70] unamed db null --- src/ngx_lua_resty_lmdb_module.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/ngx_lua_resty_lmdb_module.c b/src/ngx_lua_resty_lmdb_module.c index f862b8c8..58e6b093 100644 --- a/src/ngx_lua_resty_lmdb_module.c +++ b/src/ngx_lua_resty_lmdb_module.c @@ -6,7 +6,6 @@ #define NGX_LUA_RESTY_LMDB_VALIDATION_KEY "validation_tag" -#define NGX_LUA_RESTY_LMDB_UNNAMED_DB NULL static ngx_str_t ngx_lua_resty_lmdb_file_names[] = { @@ -409,7 +408,7 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, ngx_lua_resty_lmdb_assert(txn); - rc = mdb_dbi_open(txn, NGX_LUA_RESTY_LMDB_UNNAMED_DB, MDB_CREATE, &dbi); + rc = mdb_dbi_open(txn, NULL, MDB_CREATE, &dbi); if (rc != 0) { ngx_log_error(NGX_LOG_ERR, cycle->log, 0, "unable to open LMDB database: %s", @@ -475,7 +474,7 @@ ngx_lua_resty_lmdb_write_tag(ngx_cycle_t *cycle, ngx_lua_resty_lmdb_assert(txn); - rc = mdb_dbi_open(txn, NGX_LUA_RESTY_LMDB_UNNAMED_DB, MDB_CREATE, &dbi); + rc = mdb_dbi_open(txn, NULL, MDB_CREATE, &dbi); if (rc != 0) { ngx_log_error(NGX_LOG_ERR, cycle->log, 0, "unable to open LMDB database: %s", From 5eba04aec82e2d90f2ac4e7b49579420d500275a Mon Sep 17 00:00:00 2001 From: chronolaw Date: Tue, 14 Nov 2023 17:33:58 +0800 Subject: [PATCH 66/70] remove disabled log --- src/ngx_lua_resty_lmdb_module.c | 2 -- t/10-validation-tag-plain.t | 4 +--- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/src/ngx_lua_resty_lmdb_module.c b/src/ngx_lua_resty_lmdb_module.c index 58e6b093..17d425c3 100644 --- a/src/ngx_lua_resty_lmdb_module.c +++ b/src/ngx_lua_resty_lmdb_module.c @@ -387,8 +387,6 @@ ngx_lua_resty_lmdb_validate(ngx_cycle_t *cycle, /* check tag value in lmdb */ if (lcf->validation_tag.data == NULL) { - ngx_log_debug0(NGX_LOG_DEBUG_CORE, cycle->log, 0, - "LMDB validation disabled"); return NGX_OK; } diff --git a/t/10-validation-tag-plain.t b/t/10-validation-tag-plain.t index b780e951..b9352a87 100644 --- a/t/10-validation-tag-plain.t +++ b/t/10-validation-tag-plain.t @@ -7,7 +7,7 @@ use Cwd qw(cwd); repeat_each(1); -plan tests => repeat_each() * blocks() * 7 + 3; +plan tests => repeat_each() * blocks() * 7 + 2; my $pwd = cwd(); @@ -65,8 +65,6 @@ true value nil nil ---- error_log -LMDB validation disabled --- no_error_log [warn] [error] From 4f3e17661d073abecbedd773fef50594be82f796 Mon Sep 17 00:00:00 2001 From: chronolaw Date: Tue, 14 Nov 2023 17:39:27 +0800 Subject: [PATCH 67/70] fix goto failed error --- src/ngx_lua_resty_lmdb_module.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ngx_lua_resty_lmdb_module.c b/src/ngx_lua_resty_lmdb_module.c index 17d425c3..7c5ffb4a 100644 --- a/src/ngx_lua_resty_lmdb_module.c +++ b/src/ngx_lua_resty_lmdb_module.c @@ -501,7 +501,7 @@ ngx_lua_resty_lmdb_write_tag(ngx_cycle_t *cycle, ngx_log_error(NGX_LOG_ERR, cycle->log, 0, "unable to commit LMDB: %s", mdb_strerror(rc)); - goto failed; + return NGX_ERROR; } ngx_log_debug1(NGX_LOG_DEBUG_CORE, cycle->log, 0, From 45a4e5a9ce995f3379a14cefb205564e52608a88 Mon Sep 17 00:00:00 2001 From: chronolaw Date: Tue, 14 Nov 2023 18:05:50 +0800 Subject: [PATCH 68/70] log message --- src/ngx_lua_resty_lmdb_module.c | 2 +- t/10-validation-tag-plain.t | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/ngx_lua_resty_lmdb_module.c b/src/ngx_lua_resty_lmdb_module.c index 7c5ffb4a..cab002b2 100644 --- a/src/ngx_lua_resty_lmdb_module.c +++ b/src/ngx_lua_resty_lmdb_module.c @@ -540,7 +540,7 @@ static ngx_int_t ngx_lua_resty_lmdb_init(ngx_cycle_t *cycle) ngx_lua_resty_lmdb_close_file(cycle, lcf); ngx_log_error(NGX_LOG_WARN, cycle->log, 0, - "LMDB database tag mismatch, wiping the database"); + "LMDB validation tag mismatch, wiping the database"); /* remove lmdb files to clean data */ if (ngx_lua_resty_lmdb_remove_files(cycle, lcf) != NGX_OK) { diff --git a/t/10-validation-tag-plain.t b/t/10-validation-tag-plain.t index b9352a87..e52061f4 100644 --- a/t/10-validation-tag-plain.t +++ b/t/10-validation-tag-plain.t @@ -96,7 +96,7 @@ nil --- error_log LMDB validation enabled, using validation tag: "3.3" LMDB validation tag does not exist -LMDB database tag mismatch, wiping the database +LMDB validation tag mismatch, wiping the database set LMDB validation tag: "3.3" --- no_error_log [emerg] @@ -129,7 +129,7 @@ nil --- error_log LMDB validation enabled, using validation tag: "3.4" LMDB validation tag "3.3" did not match configured tag "3.4" -LMDB database tag mismatch, wiping the database +LMDB validation tag mismatch, wiping the database set LMDB validation tag: "3.4" --- no_error_log [emerg] From 53c4a69582c4d492773d9bfa1afe4112d8591302 Mon Sep 17 00:00:00 2001 From: Datong Sun Date: Tue, 14 Nov 2023 18:18:02 +0800 Subject: [PATCH 69/70] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5a6fc08c..9e1a7e0b 100644 --- a/README.md +++ b/README.md @@ -242,7 +242,7 @@ When LMDB starts, it will check the tag value, if the value is different from the directive value, the content of LMDB will be cleaned up. -When this directive is not set, tag is not validated. +When this directive is not set, tag validation is disabled. [Back to TOC](#table-of-contents) From e7519cf6e2cf3dd2dc07c84a1763ba0968b2e307 Mon Sep 17 00:00:00 2001 From: Datong Sun Date: Tue, 14 Nov 2023 18:20:06 +0800 Subject: [PATCH 70/70] Update src/ngx_lua_resty_lmdb_module.c --- src/ngx_lua_resty_lmdb_module.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ngx_lua_resty_lmdb_module.c b/src/ngx_lua_resty_lmdb_module.c index cab002b2..5d3222ca 100644 --- a/src/ngx_lua_resty_lmdb_module.c +++ b/src/ngx_lua_resty_lmdb_module.c @@ -499,7 +499,7 @@ ngx_lua_resty_lmdb_write_tag(ngx_cycle_t *cycle, rc = mdb_txn_commit(txn); if (rc != 0) { ngx_log_error(NGX_LOG_ERR, cycle->log, 0, - "unable to commit LMDB: %s", + "unable to commit validation tag into LMDB: %s", mdb_strerror(rc)); return NGX_ERROR; }