WIP

config

@@ -2,6 +2,7 @@ ngx_module_type=CORE
 ngx_module_name=ngx_lua_resty_lmdb_module
 ngx_module_srcs="$ngx_addon_dir/src/ngx_lua_resty_lmdb_module.c
                  $ngx_addon_dir/src/ngx_lua_resty_lmdb_transaction.c
+                 $ngx_addon_dir/src/ngx_lua_resty_lmdb_range.c
                  $ngx_addon_dir/src/ngx_lua_resty_lmdb_status.c
                  "
 ngx_module_incs="$ngx_addon_dir/lmdb/libraries/liblmdb $ngx_addon_dir/src"

lib/resty/lmdb.lua

@@ -4,6 +4,7 @@ local _M = {}
 local transaction = require("resty.lmdb.transaction")
 local status = require("resty.lmdb.status")
 
+
 do
     local CACHED_TXN = transaction.begin(1)
 

lib/resty/lmdb/ffi.lua

@@ -0,0 +1,45 @@
+local ffi = require("ffi")
+local base = require("resty.core.base")
+
+
+local DEFAULT_VALUE_BUF_SIZE = 512 * 2048 -- 1MB
+base.set_string_buf_size(DEFAULT_VALUE_BUF_SIZE)
+
+
+ffi.cdef([[
+typedef unsigned int 	MDB_dbi;
+
+
+typedef enum {
+    NGX_LMDB_OP_GET = 0,
+    NGX_LMDB_OP_PREFIX,
+    NGX_LMDB_OP_SET,
+    NGX_LMDB_OP_DB_OPEN,
+    NGX_LMDB_OP_DB_DROP
+} ngx_lua_resty_lmdb_operation_e;
+
+
+typedef struct {
+    ngx_lua_resty_lmdb_operation_e  opcode;
+    ngx_str_t                       key;   /* GET, SET */
+    ngx_str_t                       value; /* GET, SET */
+    MDB_dbi                         dbi;   /* ALL OPS */
+    unsigned int                    flags; /* SET, DROP */
+} ngx_lua_resty_lmdb_operation_t;
+
+typedef struct {
+    size_t          map_size;        /**< Size of the data memory map */
+    unsigned int    page_size;       /**< Size of a database page. */
+    unsigned int    max_readers;     /**< max reader slots in the environment */
+    unsigned int    num_readers;     /**< max reader slots used in the environment */
+    unsigned int    allocated_pages; /**< number of pages allocated */
+    size_t          in_use_pages;    /**< number of pages currently in-use */
+    unsigned int    entries;         /**< the number of entries (key/value pairs) in the environment */
+} ngx_lua_resty_lmdb_ffi_status_t;
+
+int ngx_lua_resty_lmdb_ffi_env_info(ngx_lua_resty_lmdb_ffi_status_t *lst, char **err);
+
+
+int ngx_lua_resty_lmdb_ffi_execute(ngx_lua_resty_lmdb_operation_t *ops,
+    size_t n, int need_write, unsigned char *buf, size_t buf_len, char **err);
+]])

lib/resty/lmdb/range.lua

@@ -0,0 +1,69 @@
+local _M = {}
+
+
+local ffi = require("ffi")
+local table_new = require("table.new")
+require("resty.lmdb.ffi")
+local transaction = require("resty.lmdb.transaction")
+local base = require("resty.core.base")
+
+
+local DEFAULT_OPS_SIZE = 512
+local DEFAULT_DB = transaction.DEFAULT_DB
+local C = ffi.C
+
+
+local ffi_string = ffi.string
+local get_dbi = transaction.get_dbi
+local err_ptr = base.get_errmsg_ptr()
+
+
+function _M.page(start, db)
+    local value_buf_size = get_string_buf_size()
+    local ops = ffi_new("ngx_lua_resty_lmdb_operation_t[?]", DEFAULT_OPS_SIZE)
+
+    ops[0].opcode = C.NGX_LMDB_OP_GET
+    cop.key.data = start
+    cop.key.len = #start
+    cop.dbi = get_dbi(false, db or DEFAULT_DB)
+
+::again::
+    local buf = get_string_buf(value_buf_size, false)
+    local ret = C.ngx_lua_resty_lmdb_ffi_range(ops, DEFAULT_OPS_SIZE,
+                    buf, value_buf_size, err_ptr)
+    if ret == NGX_ERROR then
+        return nil, ffi_string(err_ptr[0])
+    end
+
+    if ret == NGX_AGAIN then
+        value_buf_size = value_buf_size * 2
+        goto again
+    end
+
+    if ret == 0 then
+        -- unlikely case
+        return {}
+    end
+
+    assert(ret > 0)
+
+    local res = table_new(ret, 0)
+
+    for i = 1, DEFAULT_OPS_SIZE do
+        local cop = ops[i - 1]
+
+        assert(cop.opcode == C.NGX_LMDB_OP_PREFIX)
+
+        local pair = {
+            key = ffi_string(cop.key.data, cop.key.len),
+            value = ffi_string(cop.key.data, cop.key.len),
+        }
+
+        res[i] = pair
+    end
+
+    return res
+end
+
+
+return _M

lib/resty/lmdb/status.lua

@@ -1,3 +1,7 @@
+local _M = {}
+
+
+require("resty.lmdb.ffi")
 local ffi = require("ffi")
 local base = require("resty.core.base")
 
@@ -9,22 +13,6 @@ local NGX_ERROR = ngx.ERROR
 
 local err_ptr = base.get_errmsg_ptr()
 
-local _M = {}
-
-ffi.cdef([[
-    typedef struct {
-        size_t          map_size;        /**< Size of the data memory map */
-        unsigned int    page_size;       /**< Size of a database page. */
-        unsigned int    max_readers;     /**< max reader slots in the environment */
-        unsigned int    num_readers;     /**< max reader slots used in the environment */
-        unsigned int    allocated_pages; /**< number of pages allocated */
-        size_t          in_use_pages;    /**< number of pages currently in-use */
-        unsigned int    entries;         /**< the number of entries (key/value pairs) in the environment */
-    } ngx_lua_resty_lmdb_ffi_status_t;
-
-    int ngx_lua_resty_lmdb_ffi_env_info(ngx_lua_resty_lmdb_ffi_status_t *lst, char **err);    
-]])
-
 
 function _M.get_env_info()
     local env_status = ffi_new("ngx_lua_resty_lmdb_ffi_status_t[1]")

lib/resty/lmdb/transaction.lua

@@ -1,37 +1,12 @@
 local _M = {}
 
 
+require("resty.lmdb.ffi")
 local ffi = require("ffi")
 local base = require("resty.core.base")
 local table_new = require("table.new")
 
 
-ffi.cdef([[
-typedef unsigned int 	MDB_dbi;
-
-
-typedef enum {
-    NGX_LMDB_OP_GET = 0,
-    NGX_LMDB_OP_SET,
-    NGX_LMDB_OP_DB_OPEN,
-    NGX_LMDB_OP_DB_DROP,
-} ngx_lua_resty_lmdb_operation_e;
-
-
-typedef struct {
-    ngx_lua_resty_lmdb_operation_e  opcode;
-    ngx_str_t                       key;   /* GET, SET */
-    ngx_str_t                       value; /* GET, SET */
-    MDB_dbi                         dbi;   /* ALL OPS */
-    unsigned int                    flags; /* SET, DROP */
-} ngx_lua_resty_lmdb_operation_t;
-
-
-int ngx_lua_resty_lmdb_ffi_execute(ngx_lua_resty_lmdb_operation_t *ops,
-    size_t n, int need_write, unsigned char *buf, size_t buf_len, char **err);
-]])
-
-
 local err_ptr = base.get_errmsg_ptr()
 local get_string_buf = base.get_string_buf
 local get_string_buf_size = base.get_string_buf_size
@@ -43,8 +18,6 @@ local C = ffi.C
 local ffi_string = ffi.string
 local ffi_new = ffi.new
 local MIN_OPS_N = 16
-local DEFAULT_VALUE_BUF_SIZE = 16 * 1024 -- 16KB
-base.set_string_buf_size(DEFAULT_VALUE_BUF_SIZE)
 local NGX_ERROR = ngx.ERROR
 local NGX_AGAIN = ngx.AGAIN
 local NGX_OK = ngx.OK
@@ -55,6 +28,7 @@ _TXN_MT.__index = _TXN_MT
 local CACHED_DBI = {}
 local MDB_CREATE = 0x40000
 local DEFAULT_DB = "_default"
+_M.DEFAULT_DB = DEFAULT_DB
 
 
 function _M.begin(hint)

src/ngx_lua_resty_lmdb_module.h

@@ -21,6 +21,7 @@ typedef struct ngx_lua_resty_lmdb_conf_s ngx_lua_resty_lmdb_conf_t;
 
 typedef enum {
     NGX_LMDB_OP_GET = 0,
+    NGX_LMDB_OP_PREFIX,
     NGX_LMDB_OP_SET,
     NGX_LMDB_OP_DB_OPEN,
     NGX_LMDB_OP_DB_DROP

src/ngx_lua_resty_lmdb_range.c

@@ -0,0 +1,92 @@
+#include <ngx_lua_resty_lmdb_module.h>
+
+
+/*
+ * This function is the FFI call used for range lookups.
+ * It is very similar to `ngx_lua_resty_lmdb_ffi_execute` above,
+ * except we can only specify one key as the starting point at a time.
+ *
+ * The `ops[0]` will be the key to lookup, this function returns all keys
+ * >= `ops[0].key` and up to `n` will be returned at a time.
+ *
+ * Returns:
+ * * >= 0        - number of keys found. If return < `n`, then it is the last
+ *                 key in the map (no more keys afterward the last result)
+ * * `NGX_ERROR` - an error occurred, *err will contain the error string
+ * * `NGX_AGAIN` - `buf_len` is not enough, try again with larger `buf`
+ */
+int ngx_lua_resty_lmdb_ffi_range(ngx_lua_resty_lmdb_operation_t *ops,
+    size_t n, u_char *buf, size_t buf_len, const char **err)
+{
+    ngx_lua_resty_lmdb_conf_t      *lcf;
+    size_t                          i;
+    MDB_txn                        *txn;
+    int                             rc;
+    MDB_val                         key;
+    MDB_val                         value;
+    MDB_cursor                     *cur;
+
+    ngx_lua_resty_lmdb_assert(n >= 1);
+    ngx_lua_resty_lmdb_assert(ops[0].opcode == NGX_LMDB_OP_PREFIX);
+
+    lcf = (ngx_lua_resty_lmdb_conf_t *) ngx_get_conf(ngx_cycle->conf_ctx,
+                                                     ngx_lua_resty_lmdb_module);
+
+    if (lcf == NULL || lcf->env == NULL) {
+        *err = "no LMDB environment defined";
+        return NGX_ERROR;
+    }
+
+    txn = lcf->ro_txn;
+    rc = mdb_txn_renew(txn);
+    if (rc != 0) {
+        *err = mdb_strerror(rc);
+        return NGX_ERROR;
+    }
+
+    rc = mdb_cursor_open(txn, ops[0].dbi, &cur);
+    if (rc != 0) {
+        *err = mdb_strerror(rc);
+        mdb_txn_reset(txn);
+
+        return NGX_ERROR;
+    }
+
+    /* we always have at least one ops slot as asserted above */
+
+    for (i = 0; i < n; i++) {
+        key.mv_size = ops[i].key.len;
+        key.mv_data = ops[i].key.data;
+
+        rc = mdb_cursor_get(cur, &key, &value, i == 0 ? MDB_SET_RANGE : MDB_NEXT);
+        if (rc == 0) {
+            /* key found, copy result into buf */
+            if (value.mv_size > buf_len) {
+                mdb_cursor_close(cur);
+                mdb_txn_reset(txn);
+
+                return NGX_AGAIN;
+            }
+
+            ops[i].value.data = buf;
+            ops[i].value.len = value.mv_size;
+            ops[i].opcode = NGX_LMDB_OP_PREFIX;
+
+            buf = ngx_cpymem(buf, value.mv_data, value.mv_size);
+            buf_len -= value.mv_size;
+
+        } else if (rc == MDB_NOTFOUND) {
+            return i;
+
+        } else {
+            *err = mdb_strerror(rc);
+            goto err;
+        }
+    }
+
+err:
+    mdb_cursor_close(cur);
+    mdb_txn_reset(txn);
+
+    return NGX_ERROR;
+}

src/ngx_lua_resty_lmdb_transaction.c

@@ -1,6 +1,28 @@
 #include <ngx_lua_resty_lmdb_module.h>
 
 
+/*
+ * This function is the FFI call used for single key operations.
+ *
+ * Params:
+ * * `ops`          - operations array containing instructions to perform
+ * * `n`            - number of `ops`
+ * * `need_write`   - 1 if contains edit operations, 0 if not
+ * * `buf`          - results buf
+ * * `buf_len`      - size of buf
+ * * `err`          - err message output
+ *
+ * This function executes operations specified in `ops`. If the operation
+ * requires passing value back to the caller, then it is written to the
+ * `buf` buffer, up to `buf_len`. If the `buf` passed in is not large enough,
+ * then `NGX_AGAIN` is returned and the caller is instructed to retry with a
+ * larger buffer.
+ *
+ * Returns:
+ * * `NGX_OK`    - operations completed successfully
+ * * `NGX_ERROR` - an error occurred, *err will contain the error string
+ * * `NGX_AGAIN` - `buf_len` is not enough, try again with larger `buf`
+ */
 int ngx_lua_resty_lmdb_ffi_execute(ngx_lua_resty_lmdb_operation_t *ops,
     size_t n, int need_write, u_char *buf, size_t buf_len, const char **err)
 {