From ba6a64ade2be90c4374fd463a0dc6c6818b770f4 Mon Sep 17 00:00:00 2001 From: guerra Date: Fri, 30 Jun 2017 09:16:45 -0300 Subject: [PATCH 1/4] Implemented test for rule deployment --- tests/__init__.py | 13 ++++++- tests/env-sample | 6 ++- tests/rules/mlxRuleDeployment.r | 3 ++ tests/samples/test_rule_deploy.re | 3 ++ tests/test_rule_deployment.py | 61 +++++++++++++++++++++++++++++++ 5 files changed, 84 insertions(+), 2 deletions(-) create mode 100644 tests/rules/mlxRuleDeployment.r create mode 100644 tests/samples/test_rule_deploy.re create mode 100644 tests/test_rule_deployment.py diff --git a/tests/__init__.py b/tests/__init__.py index bd93ca7..66fc00b 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -54,8 +54,14 @@ class MetadataExtractConfig: IRODS_USER = config('IRODS_USER', default='rods') IRODS_RESC = config('IRODS_RESC', default='demoResc') IRODS_ZONE = config('IRODS_ZONE', default='tempZone') + IRODS_ETC_DIR = config('IRODS_ETC_DIR', default='/etc/irods') + IRODS_SERVER_CONFIG_FILE_NAME = config('IRODS_SERVER_CONFIG_FILE_NAME', default='server_config.json') + IRODS_SERVER_CONFIG_PATH = '{}/{}'.format(IRODS_ETC_DIR, IRODS_SERVER_CONFIG_FILE_NAME) + RULE_CACHE_DIR_NAME = config('RULE_CACHE_DIR', default='.rulecache') + RULE_CACHE_IRODS_PATH = '/{}/{}'.format(IRODS_ZONE, RULE_CACHE_DIR_NAME) IRODS_HOME_PATH = '/{}/home/{}'.format(IRODS_ZONE, IRODS_USER) - VAULT_PATH = '/var/lib/irods{}/Vault/home/{}'.format('/iRODS' if not IRODS_42 else '', IRODS_USER) + VAULT_ROOT_PATH = '/var/lib/irods{}/Vault'.format('/iRODS' if not IRODS_42 else '', IRODS_USER) + VAULT_PATH = '{}/home/{}'.format(VAULT_ROOT_PATH, IRODS_USER) IMETA_LS_NONE = 'AVUs defined for dataObj {}:\nNone\n' ILLUMINA_FOLDER_NAME = 'test_illumina_file' ILLUMINA_FILE_NAME = '{}.tar'.format(ILLUMINA_FOLDER_NAME) @@ -64,6 +70,7 @@ class MetadataExtractConfig: POPULATE_FILE_NAME = 'test_populate_file.txt' VCF_FILE_NAME = 'test_vcf_file.vcf' MANIFEST_FILE_NAME = 'test_manifest_file.xml' + RULE_DEPLOYMENT_FILE_NAME = 'test_rule_deploy.re' ILLUMINA_OBJ_PATH = '{}/{}'.format(IRODS_HOME_PATH, ILLUMINA_FILE_NAME) ILLUMINA_FOLDER_PATH = '{}/{}'.format(IRODS_HOME_PATH, ILLUMINA_FOLDER_NAME) ILLUMINA_METADATA_FILE = '{}/Data/Intensities/Offset/offsets.txt'.format(ILLUMINA_FOLDER_NAME) @@ -82,6 +89,7 @@ class MetadataExtractConfig: GET_MICROSERVICES_RULE_FILE = config('GET_MICROSERVICES_RULE_FILE', default='mlxGetMicroservices.r') POPULATE_RULE_FILE = config('POPULATE_RULE_FILE', default='mlxPopulate.r') MANIFEST_RULE_FILE = config('MANIFEST_RULE_FILE', default='mlxManifest.r') + RULE_DEPLOYMENT_RULE_FILE = config('RULE_DEPLOYMENT_RULE_FILE', default='mlxRuleDeployment.r') MSI_PACKAGE_VERSION = config('MSI_PACKAGE_VERSION', default='1.1.0') METALNX_MSIS_INSTALLED = config('METALNX_MSIS_INSTALLED').split(',') IRODS_MSIS_INSTALLED = config('IRODS_42_MSIS_INSTALLED').split(',') if IRODS_42 else config('IRODS_MSIS_INSTALLED').split(',') @@ -123,6 +131,9 @@ def call_extract_metadata_for_vcf(self, check_output=False, *args, **kwargs): call_function = _check_call_output if check_output else _call return self.call_rule_from_file(call_function, self.VCF_RULE_FILE, *args, **kwargs) + def call_rule_deployment(self, *args, **kwargs): + return self.call_rule_from_file(_check_call_output, self.RULE_DEPLOYMENT_RULE_FILE, *args, **kwargs) + def build_rule_file(self, rule_filename, *args, **kwargs): path_header_file = os.path.join(self.RULE_HEADERS_PATH, rule_filename) diff --git a/tests/env-sample b/tests/env-sample index 20c34ac..8aef87a 100644 --- a/tests/env-sample +++ b/tests/env-sample @@ -2,7 +2,9 @@ IRODS_USER=rods IRODS_RESC=demoResc IRODS_ZONE=tempZone IRODS_COLL_ABS_PATH=/tempZone/home/rods -MSI_PACKAGE_VERSION=1.2.0 +IRODS_ETC_DIR=/etc/irods +IRODS_SERVER_CONFIG_FILE_NAME=server_config.json +MSI_PACKAGE_VERSION=1.3.0 IRODS_42=True GET_VERSION_RULE_FILE=mlxGetVersion.r BAM_RULE_FILE=mlxExtractMetadataBam.r @@ -11,6 +13,8 @@ ILLUMINA_RULE_FILE=mlxExtractMetadataIllumina.r POPULATE_RULE_FILE=mlxPopulate.r MANIFEST_RULE_FILE=mlxManifest.r GET_MICROSERVICES_RULE_FILE=mlxGetMicroservices.r +RULE_DEPLOYMENT_RULE_FILE=mlxRuleDeployment.r +RULE_CACHE_DIR_NAME=.rulecache METALNX_MSIS_INSTALLED=libmsiget_illumina_meta.so,libmsiobjget_microservices.so,libmsiobjget_version.so,libmsiobjjpeg_extract.so,libmsiobjput_mdbam.so,libmsiobjput_mdmanifest.so,libmsiobjput_mdvcf.so,libmsiobjput_populate.so IRODS_MSIS_INSTALLED=libmsiobjget_http.so,libmsiobjget_irods.so,libmsiobjget_slink.so,libmsiobjput_http.so,libmsiobjput_irods.so,libmsiobjput_slink.so,libmsisync_to_archive.so,libmsi_update_unixfilesystem_resource_free_space.so IRODS_42_MSIS_INSTALLED=libmsi_update_unixfilesystem_resource_free_space.so,libmsisync_to_archive.so diff --git a/tests/rules/mlxRuleDeployment.r b/tests/rules/mlxRuleDeployment.r new file mode 100644 index 0000000..ba349ff --- /dev/null +++ b/tests/rules/mlxRuleDeployment.r @@ -0,0 +1,3 @@ +mlxRuleDeployment { + msirule_deployment(*rule_name, *rule_file_path); +} \ No newline at end of file diff --git a/tests/samples/test_rule_deploy.re b/tests/samples/test_rule_deploy.re new file mode 100644 index 0000000..9d0e941 --- /dev/null +++ b/tests/samples/test_rule_deploy.re @@ -0,0 +1,3 @@ +acPostProcForPut { + writeLine("serverLog", "Testing rules deployment\n"); +} \ No newline at end of file diff --git a/tests/test_rule_deployment.py b/tests/test_rule_deployment.py new file mode 100644 index 0000000..e5bcc57 --- /dev/null +++ b/tests/test_rule_deployment.py @@ -0,0 +1,61 @@ +import json +from os.path import join, dirname, realpath, exists +from unittest import TestCase, main + +from tests import MetadataExtractConfig, iput, irm, _call + + +def load_server_config_as_json(path): + """ + Loads the iRODS server config file as JSON + :param path: path to the server config file (typically under /etc/irods) + :return: server config file content as JSON + """ + with open(path, 'r') as server_config_fp: + server_config_json = json.loads(server_config_fp.read()) + return server_config_json + + +def get_rule_base_set_from_config(server_config_json): + """ + Finds the rule base set section in the iRODS server config file + :param server_config_json: iRODS server config file as JSON + :return: the 're_rulebase_set' JSON section (array) + """ + + # plugin_config > rule_engines > first array item > plugin_spec_config > re_rulebase_set + rule_base_set = server_config_json['plugin_configuration']['rule_engines'][0]['plugin_specific_configuration'][ + 're_rulebase_set'] + return rule_base_set + + +class TestRuleDeployment(TestCase, MetadataExtractConfig): + def setUp(self): + self.rule_file_path = join(self.VAULT_ROOT_PATH, self.RULE_CACHE_DIR_NAME, self.RULE_DEPLOYMENT_FILE_NAME) + self.rule_obj_path = '{}/{}'.format(self.RULE_CACHE_IRODS_PATH, self.RULE_DEPLOYMENT_FILE_NAME) + + self.cleanUp() + + local_rule_file = join(dirname(realpath(__file__)), 'samples', self.RULE_DEPLOYMENT_FILE_NAME) + iput(local_rule_file, self.RULE_CACHE_IRODS_PATH) # iput file into /tempZone/.rulecache + + def test_deploy_rule(self): + self.call_rule_deployment(rule_name=self.RULE_DEPLOYMENT_FILE_NAME, rule_file_path=self.rule_file_path) + self.assertTrue(exists(join(self.IRODS_ETC_DIR, self.RULE_DEPLOYMENT_FILE_NAME))) + + server_config_json = load_server_config_as_json(self.IRODS_SERVER_CONFIG_PATH) + rule_base_set = get_rule_base_set_from_config(server_config_json) + + self.assertTrue(self.RULE_DEPLOYMENT_FILE_NAME in rule_base_set) # rule must be present in the config file + self.assertEqual(self.RULE_DEPLOYMENT_FILE_NAME, rule_base_set[0]) # rule must be the first one in the list + + def tearDown(self): + self.cleanUp() + + def cleanUp(self): + irm('-rf', self.rule_obj_path) # remove file from iRODS + _call('rm', '-rf', self.rule_file_path) # remove file from local file system + _call('rm', '-rf', join(self.IRODS_ETC_DIR, self.RULE_DEPLOYMENT_FILE_NAME)) # remove rule from /etc/irods + +if __name__ == '__main__': + main() From 813de309e08722dd1d66c5ea32fea47aa953432e Mon Sep 17 00:00:00 2001 From: guerra Date: Mon, 17 Jul 2017 13:04:06 -0300 Subject: [PATCH 2/4] Implemented rule deployment microservice --- CMakeLists.txt | 5 +- VERSION | 2 +- .../libmsirule_deployment.cpp | 151 ++++++++++++++++++ 3 files changed, 155 insertions(+), 3 deletions(-) create mode 100644 microservices/msirule_deployment/libmsirule_deployment.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 56c2334..6c09619 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,6 +1,6 @@ cmake_minimum_required(VERSION 3.5.0 FATAL_ERROR) #CPACK_DEBIAN__PACKAGE_NAME -find_package(IRODS 4.2.0 EXACT REQUIRED) +find_package(IRODS) set(CMAKE_C_COMPILER ${IRODS_EXTERNALS_FULLPATH_CLANG}/bin/clang) set(CMAKE_CXX_COMPILER ${IRODS_EXTERNALS_FULLPATH_CLANG}/bin/clang++) @@ -9,7 +9,7 @@ set(CMAKE_EXE_LINKER_FLAGS_INIT "${CMAKE_EXE_LINKER_FLAGS} -stdlib=libc++") project(metalnx-msi-plugins C CXX) set(MLX_MSI_VERSION_MAJOR "1") -set(MLX_MSI_VERSION_MINOR "2") +set(MLX_MSI_VERSION_MINOR "3") set(MLX_MSI_VERSION_PATCH "0") set(MLX_MSI_VERSION "${MLX_MSI_VERSION_MAJOR}.${MLX_MSI_VERSION_MINOR}.${MLX_MSI_VERSION_PATCH}") set(MLX_HEADERS_PATH "${PROJECT_SOURCE_DIR}/microservices/core/include") @@ -50,6 +50,7 @@ set( msiobjput_mdmanifest msiobjput_mdvcf msiobjput_populate + msirule_deployment ) diff --git a/VERSION b/VERSION index ae4117d..511754b 100644 --- a/VERSION +++ b/VERSION @@ -1,3 +1,3 @@ -PLUGINVERSION=1.1.0 +PLUGINVERSION=1.3.0 PLUGINVERSIONINT="DEV" PLUGINNAME="metalnx_msi_plugins" \ No newline at end of file diff --git a/microservices/msirule_deployment/libmsirule_deployment.cpp b/microservices/msirule_deployment/libmsirule_deployment.cpp new file mode 100644 index 0000000..81ceaee --- /dev/null +++ b/microservices/msirule_deployment/libmsirule_deployment.cpp @@ -0,0 +1,151 @@ +#include "metalnx.h" +#include "metalnx_msi_version.h" +#include "rsModAVUMetadata.hpp" + +#include "rapidjson/document.h" +#include "rapidjson/writer.h" +#include "rapidjson/stringbuffer.h" +#include "rapidjson/filereadstream.h" +#include "rapidjson/filewritestream.h" +#include "rapidjson/prettywriter.h" + +#include +#include +#include +#include + +#include + +#define MSI_LOG "[Metalnx Rule Deployment MSI]" + +#define SERVER_CONFIG_FILE "/etc/irods/server_config.json" +#define RULE_DEST_DIR "/etc/irods" +#define PLUGIN_CONFIG_SECTION "plugin_configuration" +#define RULE_ENGINES_SECTION "rule_engines" +#define PLUGIN_SPECIFIC_CONFIG_SECTION "plugin_specific_configuration" +#define RULEBASE_SET_SECTION "re_rulebase_set" +#define RULE_FILE_EXTENSION ".re" +#define BUFFER_SIZE 65536 + +using namespace boost::filesystem; +using namespace rapidjson; + +using boost::property_tree::ptree; +using boost::property_tree::read_json; +using boost::property_tree::write_json; + +extern "C" { + + void add_rule_to_base_set(char* rule_filename) { + rodsLog(LOG_NOTICE, "%s Add rule to base set [%s]\n", MSI_LOG, rule_filename, SERVER_CONFIG_FILE); + + FILE* fp = fopen(SERVER_CONFIG_FILE, "rb"); + char readBuffer[BUFFER_SIZE]; + FileReadStream is(fp, readBuffer, sizeof(readBuffer)); + + Document document; + document.ParseStream(is); + + fclose(fp); + + rodsLog(LOG_NOTICE, "%s Adding rule [%s] to the server config base set [%s]\n", MSI_LOG, rule_filename, SERVER_CONFIG_FILE); + Document::AllocatorType& allocator = document.GetAllocator(); + Value& array = document["plugin_configuration"]["rule_engines"][0]["plugin_specific_configuration"]["re_rulebase_set"]; + array.PushBack(StringRef(""), allocator); + for (SizeType i = array.Size()-1; i > 0u; --i) + array[i] = array[i-1]; + array[0] = StringRef(rule_filename); + + rodsLog(LOG_NOTICE, "%s Saving new server config file\n", MSI_LOG); + fp = fopen(SERVER_CONFIG_FILE, "wb"); + char writeBuffer[BUFFER_SIZE]; + FileWriteStream os(fp, writeBuffer, sizeof(writeBuffer)); + PrettyWriter writer(os); + document.Accept(writer); + fclose(fp); + rodsLog(LOG_NOTICE, "%s Server config file saved\n", MSI_LOG); + + rodsLog(LOG_NOTICE, "%s Added rule [%s] to the server config base set [%s]\n", MSI_LOG, rule_filename, SERVER_CONFIG_FILE); + } + + /** + * Copies a file from a source path in iRODS to a destination path. + */ + bool copy_file_from_irods(const char* src_path, const char* dest_path) { + rodsLog(LOG_NOTICE, "%s Copying file from [%s] to [%s]\n", MSI_LOG, src_path, dest_path); + + path from (src_path); + + if (!is_regular_file(from)) { + rodsLog(LOG_ERROR, "%s Source file [%s] is not a regular file\n", MSI_LOG, src_path); + return false; + } + + std::ifstream src(src_path, std::ios::binary); + std::ofstream dst(dest_path, std::ios::binary); + + dst << src.rdbuf(); + + rodsLog(LOG_NOTICE, "%s Copied file from [%s] to [%s]\n", MSI_LOG, src_path, dest_path); + return true; + } + + const char* get_rule_dst_path(char* rule_name) { + std::string rule_dst_dir (RULE_DEST_DIR); + std::string rule_dst_file (rule_name); + std::string rule_dst_path = rule_dst_dir + "/" + rule_dst_file; + return rule_dst_path.c_str(); + } + + int msirule_deployment(msParam_t* inRuleNameParam, msParam_t* inRuleFilePathParam, ruleExecInfo_t* rei ) { + rodsLog(LOG_NOTICE, "%s MSI Rule Deployment [%s]\n", MSI_LOG, MSI_VERSION); + + char* rule_name = (char*) inRuleNameParam->inOutStruct; + char* rule_src_path = (char*) inRuleFilePathParam->inOutStruct; + + rodsLog(LOG_NOTICE, "%s Deploying rule [%s]\n", MSI_LOG, rule_name); + + // copy the rule file from the source path into the /etc/irods directory + std::string rule_dst_dir (RULE_DEST_DIR); + std::string rule_dst_file (rule_name); + rule_dst_file = rule_dst_file + RULE_FILE_EXTENSION; + std::string rule_dst_path = rule_dst_dir + "/" + rule_dst_file; + if(!copy_file_from_irods(rule_src_path, rule_dst_path.c_str())) { + return MSI_ERROR; + } + + // add the new rule into the server config file + add_rule_to_base_set(rule_name); + + return MSI_SUCCESS; + } + + // =-=-=-=-=-=-=- + // plugin factory + irods::ms_table_entry* plugin_factory( ) { + // =-=-=-=-=-=-=- + // instantiate a new msvc plugin + irods::ms_table_entry* msvc = new irods::ms_table_entry( 2 ); + + // =-=-=-=-=-=-=- + // wire the implementation to the plugin instance + msvc->add_operation< + msParam_t*, + msParam_t*, + ruleExecInfo_t*>("msirule_deployment", + std::function(msirule_deployment)); + + // =-=-=-=-=-=-=- + // hand it over to the system + return msvc; + + } // plugin_factory + +} // extern "C" + + + + From dcaa5b54185830a47b1f9b4f35afcb5b4ea96025 Mon Sep 17 00:00:00 2001 From: guerra Date: Mon, 17 Jul 2017 13:14:10 -0300 Subject: [PATCH 3/4] Updated rule test to remove extension from server config --- tests/test_rule_deployment.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_rule_deployment.py b/tests/test_rule_deployment.py index e5bcc57..2655980 100644 --- a/tests/test_rule_deployment.py +++ b/tests/test_rule_deployment.py @@ -40,14 +40,14 @@ def setUp(self): iput(local_rule_file, self.RULE_CACHE_IRODS_PATH) # iput file into /tempZone/.rulecache def test_deploy_rule(self): - self.call_rule_deployment(rule_name=self.RULE_DEPLOYMENT_FILE_NAME, rule_file_path=self.rule_file_path) + self.call_rule_deployment(rule_name=self.RULE_DEPLOYMENT_FILE_NAME[:-3], rule_file_path=self.rule_file_path) self.assertTrue(exists(join(self.IRODS_ETC_DIR, self.RULE_DEPLOYMENT_FILE_NAME))) server_config_json = load_server_config_as_json(self.IRODS_SERVER_CONFIG_PATH) rule_base_set = get_rule_base_set_from_config(server_config_json) - self.assertTrue(self.RULE_DEPLOYMENT_FILE_NAME in rule_base_set) # rule must be present in the config file - self.assertEqual(self.RULE_DEPLOYMENT_FILE_NAME, rule_base_set[0]) # rule must be the first one in the list + self.assertTrue(self.RULE_DEPLOYMENT_FILE_NAME[:-3] in rule_base_set) # rule must be present in the config file + self.assertEqual(self.RULE_DEPLOYMENT_FILE_NAME[:-3], rule_base_set[0]) # rule must be the first one in the list def tearDown(self): self.cleanUp() From 9217ef5f583f07e5f370e98fe98e3026370ff700 Mon Sep 17 00:00:00 2001 From: guerra Date: Mon, 17 Jul 2017 13:55:35 -0300 Subject: [PATCH 4/4] Included rapidjson --- CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6c09619..cd7d21e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -20,6 +20,7 @@ configure_file ( "${MLX_CMAKE_HEADERS_PATH}/metalnx_msi_version.h" ) +include_directories(/usr/include/rapidjson) include_directories(/usr/include/libxml2) include_directories(${IRODS_EXTERNALS_FULLPATH_CLANG}/include/c++/v1) include_directories("${MLX_CMAKE_HEADERS_PATH}")