From 45569fd74078c30e823874276619e4672ede462f Mon Sep 17 00:00:00 2001 From: Attila Kovacs Date: Wed, 11 Dec 2024 22:35:56 +0100 Subject: [PATCH] New DelKey.lua, and initial smax / dsm function libs --- install.sh | 4 +- lua/DelKey.lua | 44 +++++ lua/DelStruct.lua | 6 - lua/dsm.lib | 38 +++++ lua/smax.lib | 367 +++++++++++++++++++++++++++++++++++++++++ smax-init.sh | 1 + smax-load-functions.sh | 39 +++++ 7 files changed, 491 insertions(+), 8 deletions(-) create mode 100644 lua/DelKey.lua delete mode 100644 lua/DelStruct.lua create mode 100644 lua/dsm.lib create mode 100644 lua/smax.lib create mode 100755 smax-load-functions.sh diff --git a/install.sh b/install.sh index 4a441d9..dd8e893 100755 --- a/install.sh +++ b/install.sh @@ -78,7 +78,7 @@ sed -i "s:/usr:$DESTDIR:g" $SYSTEMD/smax-scripts.service if [[ ! $1 =~ ^(sma|SMA)$ ]] ; then echo ". Removing SMA-specific sections from scripts" - sed -i '/^.*BEGIN SMA.*/,/^.*END SMA.*/d' $SMAX/lua/*.lua + sed -i '/^.*BEGIN SMA.*/,/^.*END SMA.*/d' $SMAX/lua/*.lua $SMAX/lua/*.lib fi # Register smax-scripts with systemd @@ -142,7 +142,7 @@ else echo # (optional) move to a new line if [[ $REPLY =~ ^[Yy]$ ]] ; then echo ". Removing SMA-specific sections from scripts" - sed -i '/^.*BEGIN SMA.*/,/^.*END SMA.*$/d' *.lua + sed -i '/^.*BEGIN SMA.*/,/^.*END SMA.*$/d' *.lua *.lib fi read -p "start redis with SMA-X scripts at this time? " -n 1 -r diff --git a/lua/DelKey.lua b/lua/DelKey.lua new file mode 100644 index 0000000..cd0d662 --- /dev/null +++ b/lua/DelKey.lua @@ -0,0 +1,44 @@ +-- keys: [1+] SMA-X keywords +-- arguments: (none) +-- returns: (integer) the total number of fields deleted, including in sub-structures, and in parent structures. + +local metas = { '', '', '', '', '', '', '', '', '' } +local n = 0 + +local function DelKey (table) + -- Recursively delete table entries + for f in redis.call('hkeys', table) do + DelKey(table..':'..field) + end + + -- Delete metadata for the table + for m in metas do + redis.call("hdel", m, table) + end + + -- Delete the table itself + if redis.call('del', table) == 1 then + n = n + 1 + end +end + +-- Process each input keyword +for key in KEYS do + -- Delete the table (if any) recuresively + DelKey(key) + + -- match the substring starting with the last : + local tail = key:gmatch(':(?:.(?!:))+') + + -- If the keyword can be split... + if tail ~= nil and tail ~= '' then + -- Delete reference from parent table + local parent = table:sub(1, -tail:len()) + local ref = tail:sub(2) + if redis.call('hdel', parent, ref) == 1 then + n = n + 1 + end + end +end + +return n diff --git a/lua/DelStruct.lua b/lua/DelStruct.lua deleted file mode 100644 index f35d515..0000000 --- a/lua/DelStruct.lua +++ /dev/null @@ -1,6 +0,0 @@ --- keys: [1] Structure name (hash table) --- arguments: (none) --- returns: (integer) the number of keys deleted, including sub-structures. - -local structs = redis.call('keys', KEYS[1] .. '*') -return redis.call('del', unpack(structs)) diff --git a/lua/dsm.lib b/lua/dsm.lib new file mode 100644 index 0000000..f50a55b --- /dev/null +++ b/lua/dsm.lib @@ -0,0 +1,38 @@ +#!lua name=dsm + +-- Legacy DSM emulation helper library for Redis +-- Author: Attila Kovacs +-- Version: 11 December 2024 +-- +-- GitHub: Smithsonian/smax-server + +local function dsm_get_key(KEYS, ARGS) + -- keys: + -- arguments: host target key + -- returns name SMA-X table name under which the data can be found + + local table = "DSM:"..ARGV[2] + local key = ARGV[3] + + -- If the data is stored under the target name, use that + if redis.call('hexists', table, key) == 1 then + return table + end + + -- If the data is stored under the caller's name, use that + table = "DSM:"..ARGV[1] + if redis.call('hexists', table, key) == 1 then + return table + end + + -- LUA false maps to Redis nil + return false +end + + +redis.register_function { + function_name='dsm_get_key', + callback=dsm_get_key, + flags={ 'no-writes' }, + description='(|host, target, key) Returns the SMA-X table for the given host and target machine and DSM key' +} diff --git a/lua/smax.lib b/lua/smax.lib new file mode 100644 index 0000000..9cecbc0 --- /dev/null +++ b/lua/smax.lib @@ -0,0 +1,367 @@ +#!lua name=smax + +local function smax_set(KEYS, ARGS) + -- keys: [1] Hash table to add value to + -- arguments: 1:origin 2:field 3:value 4:type 5:dim + -- returns the result of the HSET call for the {field,value} pair + local table = KEYS[1] + local origin = ARGV[1] + local field = ARGV[2] + local value = ARGV[3] + + -- Timestamp + local time = redis.call('time') + local timestamp = time[1].."."..string.format("%06d", time[2]) + + -- Set the key/value + local result = redis.call('hset', table, field, value) + + -- Set the corresponding metadata + local id = table .. ':' .. field + redis.call('hset', '', id, ARGV[4]) + redis.call('hset', '', id, ARGV[5]) + redis.call('hset', '', id, timestamp) + redis.call('hset', '', id, origin) + redis.call('hincrby', '', id, '1') + + -- Send notification for the table update + redis.call('publish', 'smax:'..id, origin) + + -- <======== BEGIN SMA-specific section ========> + -- Check if updating RM variables... + if table:sub(1, 3) == 'RM:' then + -- Send RM update data over PUB/SUB... + redis.call('publish', '@'..id, value) + -- Check if data is from an rm2smax replicator + if origin:find(':rm2smax') == nil then + -- Send RM insert notification over PUB/SUB + redis.call('publish', id, value) + end + end + -- <======== END SMA-specific section ========> + + -- Add/update the parent hierachy + local parent = '' + for child in table:gmatch('[^:]+') do + if parent == '' then + parent = child + else + id = parent..':'..child + + redis.call('hset', parent, child, id) + redis.call('hset', '', id, 'struct') + redis.call('hset', '', id, '1') + redis.call('hset', '', id, timestamp) + redis.call('hset', '', id, origin) + + parent = id + end + end + + return result +end + + + +local function smax_get(KEYS, ARGS) + -- keys: [1] Hash table to get value from + -- arguments: field + -- returns an array of { value, type, dim, timestamp, origin, serial } + local table = KEYS[1] + local field = ARGV[1] + local value = redis.call('hget', table, field) + local id = table .. ':' .. field + local vtype = redis.call('hget', '', id) + local dim = redis.call('hget', '', id) + local timestamp = redis.call('hget', '', id) + local origin = redis.call('hget', '', id) + local serial = redis.call('hget', '', id) + return { value, vtype, dim, timestamp, origin, serial } +end + + + +local function smax_mset(KEYS, ARGS) + -- keys: [1] Hash table to add values to + -- arguments: 1:origin {2:field1 3:value1 4:type1 5:dim1 } { 6:field2 ... } [T]... + -- returns the result of the HMSET call for the {field,value} pairs + local table = KEYS[1] -- the name of the hash table + local origin = ARGV[1] + + -- Timestamp + local time = redis.call('time') + local timestamp = time[1].."."..string.format("%06d", time[2]) + + -- arrays / tables we'll need + local ids = {} -- composited meta IDs for all fields to be set + + -- Pairs to pass to HMSET + local entries = {} -- {field,value} pairs + local types = {} -- {field,vartype} pairs + local dims = {} -- {field,vardim} pairs + local timestamps = {} -- {field,timestamp} pairs + local origins = {} -- {field,origin} pairs + local serials = {} -- {field,serial} pairs + + local leadArgs = 1 + local N = math.floor((#ARGV - leadArgs) / 4) -- Number of fields + + -- A trailing 'T' argument specifies that this is a top-level structure + -- (not a nested substructure component in a series of updates) + local isNested = (ARGV[leadArgs + 4 * N + 1] ~= 'T') + + for k=1,N do + local i = leadArgs + 4*k-3 -- i is the original ARGV index + local j = 2 * k - 1 -- j is the {key,value} pairs list index + + local field = ARGV[i] -- field name + local id = table..':'..field -- the composited meta id + + ids[k] = id + + entries[j] = field -- set the pair names... + types[j] = id + dims[j] = id + timestamps[j] = id + origins[j] = id + serials[j] = id + + j = j+1 -- set the pair values... + entries[j] = ARGV[i+1] -- value + types[j] = ARGV[i+2] -- type + dims[j] = ARGV[i+3] -- dim + timestamps[j] = timestamp + origins[j] = origin + end + + local result = redis.call('hset', table, unpack(entries)) + redis.call('hset', '', unpack(types)) + redis.call('hset', '', unpack(dims)) + redis.call('hset', '', unpack(timestamps)) + redis.call('hset', '', unpack(origins)) + + -- Bulk update of the serial counters + local counts = redis.call('hmget', '', unpack(ids)) + for i,ser in pairs(counts) do + if ser then + serials[2*i] = tonumber(ser) + 1 + else + serials[2*i] = 1 + end + end + + redis.call('hset', '', unpack(serials)) + + local from = origin; + + if isNested then + -- If thus is not a top-level structure, tag the message body with '' + from = origin .. ' ' + end + + -- Notify of the table update itself. + redis.call('publish', 'smax:'..table, from) + + -- <======== BEGIN SMA-specific section ========> + -- Check if updating RM variables... + local isRM = table:sub(1, 3) == 'RM:' + local isExternalRM = false + if isRM then + -- Check if data is from an rm2smax replicator + isExternalRM = (origin:find(':rm2smax') == nil) + end + -- <======== END SMA-specific section ========> + + -- Tag leaf updates with '' + local from = origin .. ' ' + + -- Send notification of all updated leafs also... + for i,id in ipairs(ids) do + redis.call('publish', 'smax:'..id, from) + + -- <======== BEGIN SMA-specific section ========> + if isRM then + local value = entries[2*i]; + -- Send RM update data over PUB/SUB... + redis.call('publish', '@'..id, value) + -- For RM updates not coming from an `rm2smax` replicator, send an + -- update notification (to an rm2smax replicator) + if isExternalRM then + -- Send RM insert notification over PUB/SUB + redis.call('publish', id, value) + end + end + -- <======== END SMA-specific section ========> + end + + -- If it's a nested sub-structure, then there isn't anything left to do. + if isNested then + return + end + + -- Add/uppdate the parent hierachy as needed + local stem = '' + for token in table:gmatch('[^:]+') do + if stem == '' then + stem = token + else + local id = stem..':'..token + + redis.call('hset', stem, token, id) + redis.call('hset', '', id, 'struct') + redis.call('hset', '', id, '1') + redis.call('hset', '', id, timestamp) + redis.call('hset', '', id, origin) + + stem = id + end + end + + return result +end + + + + +local function smax_mget(KEYS, ARGS) + -- keys: [1] Hash table to get values from + -- arguments: field1 field2 ... + -- returns an array of arrays { {values}, {types}, {dims}, {timestamps}, {origins}, {serials} } + local table = KEYS[1] + + local values = redis.call('hmget', table, unpack(ARGV)) + local ids = {} + + for i,field in ipairs(ARGV) do + ids[i] = table..':'..field + end + + local vtypes = redis.call('hmget', '', unpack(ids)) + local dims = redis.call('hmget', '', unpack(ids)) + local timestamps = redis.call('hmget', '', unpack(ids)) + local origins = redis.call('hmget', '', unpack(ids)) + local serials = redis.call('hmget', '', unpack(ids)) + return { values, vtypes, dims, timestamps, origins, serials } +end + + + + +local function smax_get_struct(KEYS) + -- keys: [1] Structure name (hash table) + -- arguments: (none) + -- Arrays of: + -- { + -- { struct/substruct names }, -- (Bulk strings) names of structures and nested substructures to follow in order + -- { field names }, { { values}, {types}, {dims}, {timestamps}, {origins}, {serials} }, -- data for first structure name + -- ... -- further nested structure data + -- } + + local table = KEYS[1] -- the name of the hash table + + local id = KEYS[1] + local structs = redis.call('keys', id..'*') + local result = {} + result[1] = structs + for i,table in ipairs(structs) do + local k = 2*i + local keys = redis.call('hkeys', table); + result[k] = keys + result[k+1] = smax_mget(table, keys) + end + return result +end + + + + +local function smax_del_key(table) + -- Recursively delete table entries + for f in redis.call('hkeys', table) do + smax_del_key(table..':'..field) + end + + -- Delete metadata for the table + for m in metas do + redis.call("hdel", m, table) + end + + -- Delete the table itself + if redis.call('del', table) == 1 then + n = n + 1 + end +end + + + + +local function smax_del(KEYS) + -- keys: [1+] SMA-X keywords + -- arguments: (none) + -- returns: (integer) the total number of fields deleted, including in sub-structures, and in parent structures. + + local metas = { '', '', '', '', '', '', '', '', '' } + local n = 0 + + -- Process each input keyword + for key in KEYS do + -- Delete the table (if any) recuresively + smax_del_key(key) + + -- match the substring starting with the last : + local tail = key:gmatch(':(?:.(?!:))+') + + -- If the keyword can be split... + if tail ~= nil and tail ~= '' then + -- Delete reference from parent table + local parent = table:sub(1, -tail:len()) + local ref = tail:sub(2) + if redis.call('hdel', parent, ref) == 1 then + n = n + 1 + end + end + end + + return n +end + +redis.register_function{ + function_name='smax_set', + callback=smax_set, + description='(table | origin, key, value, type, dim) Sets an SMA-X leaf node' +} + +redis.register_function{ + function_name='smax_get', + callback=smax_get, + flags={ 'no-writes' }, + description='(table | key) Returns an SMA-X variable (table:key) with metadata' +} + +redis.register_function{ + function_name='smax_mset', + callback=smax_mset, + flags={}, + description='(table | origin { key1, value1, type1, dim1 } ...) Sets multiple SMA-X entrie in the same table' +} + +redis.register_function{ + function_name='smax_mget', + callback=smax_mget, + flags={ 'no-writes' }, + description='(table | key1 ...) Returns multiple SMA-X entries from a table with metadata' +} + +redis.register_function{ + function_name='smax_get_struct', + callback=smax_get_struct, + flags={ 'no-writes' }, + description='(table) Returns an entire structure from SMA-X atomically with metadata' +} + +redis.register_function{ + function_name='smax_del', + callback=smax_del, + flags={}, + description='Deletes a SMA-X ID, removing all references in parents and metadata alike' +} diff --git a/smax-init.sh b/smax-init.sh index 8ee0f39..0375ad9 100755 --- a/smax-init.sh +++ b/smax-init.sh @@ -47,6 +47,7 @@ load_script HMGetWithMeta load_script HMSetWithMeta load_script GetStruct load_script DSMGetTable +load_script DelKey exit 0 diff --git a/smax-load-functions.sh b/smax-load-functions.sh new file mode 100755 index 0000000..89dcef4 --- /dev/null +++ b/smax-load-functions.sh @@ -0,0 +1,39 @@ +#!/bin/bash +# +# Loads the SMA-X function into Redis. +# +# Author: Attila Kovacs +# Version: 2024 December 11 + +LUA="/usr/share/smax/lua" + +if [ "$1" != "" ] ; then + LUA="$1" +fi + +# Try for up to 5 seconds to get a response from redis... +for i in {1..5}; do + result=`redis-cli ping` + if [ "$result" == "PONG" ] ; then + break + fi + if [ $i -eq 5 ]; then + echo "ERROR! Could not connect to Redis. SMA-X scripts not loaded." + exit 1 + fi + sleep 1 +done + +echo "INFO: Redis is online. Loading SMA-X functions..." + +load_function() { + NAME=$1 + echo -n "> Loading $NAME. New? " + `cat $LUA/$NAME | redis-cli function load replace` +} + +load_function smax.lib + +exit 0 + +