From 040e098eabb9552a358f3147031a3d271600c565 Mon Sep 17 00:00:00 2001 From: Sergei Pond Date: Mon, 28 Jan 2019 13:27:42 -0500 Subject: [PATCH] Bug fixes --- res/TemplateBatchFiles/SM-2019.bf | 26 +++++++------------ res/TemplateBatchFiles/libv3/tasks/trees.bf | 17 +++++++++++- src/core/batchlan.cpp | 11 +++++--- src/core/batchlanruntime.cpp | 14 ++++------ src/core/global_things.cpp | 9 +++++++ src/core/matrix.cpp | 2 +- src/mains/unix.cpp | 17 ++++-------- src/utils/hyphyunixutils.cpp | 6 ++--- .../hbltests/UnitTests/HBLCommands/Random.bf | 10 +++---- tests/hbltests/libv3/BUSTED.wbf | 2 +- 10 files changed, 60 insertions(+), 54 deletions(-) diff --git a/res/TemplateBatchFiles/SM-2019.bf b/res/TemplateBatchFiles/SM-2019.bf index 268548f9d..6ae35c960 100644 --- a/res/TemplateBatchFiles/SM-2019.bf +++ b/res/TemplateBatchFiles/SM-2019.bf @@ -53,8 +53,9 @@ utility.ForEachPair (sm.partitions , "_key_", "_value_", ' '); -sm.bootstrap = trees.BootstrapSupport (sm.tree); +sm.replicates = io.PromptUser(">How many bootstrap replicates", 100, 1, 1000000, TRUE); +sm.bootstrap = trees.BootstrapSupport (sm.tree); sm.bootstrap_weighting = {"default" : 0.2}; if (utility.Array1D (sm.bootstrap )) { @@ -68,16 +69,8 @@ if (utility.Array1D (sm.bootstrap )) { } -/* -sm.method = io.SelectAnOption ({"Standard" : "Permute leaf labels freely (unconstrained)", - "Restricted" : "Permute leaf labels in subtrees defined by randomly chosen internal branches (rate 1/4)"}, - "Resampling method" - ); -*/ -sm.replicates = io.PromptUser(">How many bootstrap replicates", 100, 1, 1000000, TRUE); - sm.tree.string = sm.tree[terms.trees.newick_with_lengths]; sm.node_labels = {}; @@ -95,13 +88,8 @@ utility.ForEachPair (sm.partitions, "_regexp_", "_leaves_", ); -//console.log (T); -//console.log (sm.node_labels); - -sm.mp = //trees.ParsimonyLabel ("T", sm.node_labels); -Max (T, {"labels": sm.node_labels}); -//console.log (sm.mp); +sm.mp = Max (T, {"labels": sm.node_labels}); sm.score = sm.mp ["score"]; io.ReportProgressMessageMD('SM', 'result', 'Inferred **' + sm.score + '** migration events'); @@ -148,7 +136,6 @@ sm.dist.structured.cutoff = sm.dist.structured [sm.replicates * 90 $ 100]; sm.structured.p = +sm.dist.shuffled["_MATRIX_ELEMENT_VALUE_<= sm.dist.structured.cutoff"]; - sm.sampled_stats = math.GatherDescriptiveStats (sm.resampled_distribution[-1][0]); sm.sampled_stuctured_stats = math.GatherDescriptiveStats (sm.resampled_distribution[-1][1]); @@ -167,6 +154,9 @@ io.ReportProgressMessageMD('SM', 'reshuffled', 'Based on **' + sm.replicates + + Format ( sm.structured.p /(sm.replicates+1), 6, 3)); io.ReportProgressMessageMD('SM', 'reshuffled', '\n> This p-value is derived by comparing the distribution of migration events from the panmictic reshuffle to the 90% percentile of the simulated distribution of expected migrations if leaf labels are permuted partially respecting subtree structure (block permutations), which results in **' + sm.dist.structured.cutoff + '** expected migrations'); +sm.json_path = sm.tree[terms.data.file] + ".json"; +io.ReportProgressMessageMD('SM', 'file', 'Saving detailed report as a JSON file to \`' + sm.json_path + '\`'); + sm.json = { 'replicates' : sm.replicates, 'compartments' : sm.group_count, @@ -178,7 +168,9 @@ sm.json = { 'simulations' : { 'panmictic' : sm.dist.shuffled, 'structured' : sm.dist.structured - } + }, + "events" : sm.mp["substitutions"] }; +io.SpoolJSON (sm.json, sm.json_path); return sm.json; \ No newline at end of file diff --git a/res/TemplateBatchFiles/libv3/tasks/trees.bf b/res/TemplateBatchFiles/libv3/tasks/trees.bf index 9190903f0..ee4ebaf60 100644 --- a/res/TemplateBatchFiles/libv3/tasks/trees.bf +++ b/res/TemplateBatchFiles/libv3/tasks/trees.bf @@ -141,6 +141,8 @@ lfunction trees.GetTreeString(look_for_newick_tree) { SetDialogPrompt("Please select a tree file for the data:"); fscanf(PROMPT_FOR_FILE, REWIND, "Raw", treeString); fprintf(stdout, "\n"); + + look_for_newick_tree = utility.getGlobalValue ("LAST_FILE_PATH"); if (regexp.Find(treeString, "^#NEXUS")) { ExecuteCommands(treeString); @@ -196,7 +198,11 @@ lfunction trees.GetTreeString(look_for_newick_tree) { } } - return treeString; + return + { + utility.getGlobalValue("terms.data.file"): look_for_newick_tree, + utility.getGlobalValue("terms.data.tree"): treeString + }; } @@ -379,6 +385,13 @@ lfunction trees.RootTree(tree_info, root_on) { */ lfunction trees.ExtractTreeInfo(tree_string) { + if (Type (tree_string) == "AssociativeList") { + file_name = tree_string[^"terms.data.file"]; + tree_string = tree_string[^"terms.data.tree"]; + } else { + file_name = None; + } + Topology T = tree_string; branch_lengths = BranchLength(T, -1); @@ -418,6 +431,8 @@ lfunction trees.ExtractTreeInfo(tree_string) { ^"terms.trees.model_list": Columns(modelMap), ^"terms.trees.rooted" : rooted, ^"terms.trees.meta" : T.__meta, + ^"terms.data.file" : file_name + }; } diff --git a/src/core/batchlan.cpp b/src/core/batchlan.cpp index 29a4be8a2..e19be0b4f 100644 --- a/src/core/batchlan.cpp +++ b/src/core/batchlan.cpp @@ -263,7 +263,7 @@ void MPISendString (_String const& theMessage, long destID, bool isErro { long messageLength = theMessage.length(), - transferCount = 0; + transferCount = 0L; if (isError) { messageLength = -messageLength; @@ -271,7 +271,7 @@ void MPISendString (_String const& theMessage, long destID, bool isErro ReportMPIError(MPI_Send(&messageLength, 1, MPI_LONG, destID, HYPHY_MPI_SIZE_TAG, MPI_COMM_WORLD),true); - if (messageLength == 0) { + if (messageLength == 0L) { return; } @@ -300,8 +300,8 @@ void MPISendString (_String const& theMessage, long destID, bool isErro //____________________________________________________________________________________ _String* MPIRecvString (long senderT, long& senderID) { _String* theMessage = nil; - long messageLength = 0, - transferCount = 0; + long messageLength = 0L, + transferCount = 0L; int actualReceived = 0; bool isError = false; @@ -329,6 +329,8 @@ _String* MPIRecvString (long senderT, long& senderID) { isError = true; messageLength = -messageLength; } + + //printf ("MPIRecvString size tag %ld (size chunk %ld) \n",messageLength, MPI_SEND_CHUNK); if (!isError) { //MPI_Get_count (&status,MPI_CHAR,&actualReceived); @@ -352,6 +354,7 @@ _String* MPIRecvString (long senderT, long& senderID) { } if (messageLength-transferCount) { + //printf ("Clause 2 %d %d\n", messageLength-transferCount, theMessage->length()); ReportMPIError(MPI_Recv((void*)(theMessage->get_str()+transferCount), messageLength-transferCount, MPI_CHAR, senderT, HYPHY_MPI_STRING_TAG, MPI_COMM_WORLD,&status),false); MPI_Get_count (&status,MPI_CHAR,&actualReceived); if (actualReceived!=messageLength-transferCount) { diff --git a/src/core/batchlanruntime.cpp b/src/core/batchlanruntime.cpp index a1f521742..e16af4126 100644 --- a/src/core/batchlanruntime.cpp +++ b/src/core/batchlanruntime.cpp @@ -1969,21 +1969,17 @@ bool _ElementaryCommand::HandleMPIReceive (_ExecutionList& current_program) #ifdef __HYPHYMPI__ - - - receptacle = _ValidateStorageVariable (current_program, 1UL); _Variable* node_index_storage = _ValidateStorageVariable (current_program, 2UL); long target_node = _ProcessNumericArgumentWithExceptions(*GetIthParameter(0UL), current_program.nameSpacePrefix), - node_count = hy_env::EnvVariableGetDefaultNumber(hy_env::mpi_node_count); + node_count = hy_env::EnvVariableGetNumber(hy_env::mpi_node_count); if (target_node < -1L || target_node >= node_count) { - throw (GetIthParameter(1UL)->Enquote () & " (=" & node_count & ") is not a valid MPI node index (or -1 to accept from any node"); + throw (GetIthParameter(1UL)->Enquote () & " (=" & node_count & ") must be a valid MPI node index (or -1 to accept from any node"); } long received_from; - receptacle->SetValue(new _FString (MPIRecvString (target_node,received_from)), false); node_index_storage->SetValue (new _Constant (received_from), false); @@ -2009,10 +2005,10 @@ bool _ElementaryCommand::HandleMPISend (_ExecutionList& current_program){ #ifdef __HYPHYMPI__ long target_node = _ProcessNumericArgumentWithExceptions(*GetIthParameter(0UL), current_program.nameSpacePrefix), - node_count = hy_env::EnvVariableGetDefaultNumber(hy_env::mpi_node_count); + node_count = hy_env::EnvVariableGetNumber(hy_env::mpi_node_count); if (target_node < 0L || target_node >= node_count) { - throw (GetIthParameter(1UL)->Enquote () & " (=" & node_count & ") is not a valid MPI node index"); + throw (GetIthParameter(1UL)->Enquote () & " (=" & node_count & ") is not a valid MPI node index; valud range is " & target_node & " to " & (node_count-1)); } _StringBuffer message_to_send (1024UL); @@ -2039,7 +2035,7 @@ bool _ElementaryCommand::HandleMPISend (_ExecutionList& current_program){ if (message_to_send.nonempty()) { MPISendString(message_to_send, target_node); } else { - throw ("An ivalid (empty) MPI message"); + throw (_String ("An invalid (empty) MPI message")); } #else throw ("Command not supported for non-MPI versions of HyPhy. HBL scripts need to check for MPI before calling MPI features"); diff --git a/src/core/global_things.cpp b/src/core/global_things.cpp index e446e2049..5374d4b67 100644 --- a/src/core/global_things.cpp +++ b/src/core/global_things.cpp @@ -57,6 +57,8 @@ using namespace hy_env; +#define __HYPHY_MPI_MESSAGE_LOGGING__ + extern _SimpleList freeSlots; @@ -356,6 +358,13 @@ namespace hy_global { has_terminal_stdout = isatty (STDOUT_FILENO); has_terminal_stderr = isatty (STDERR_FILENO); + + +#ifdef __HYPHYMPI__ + hy_env :: EnvVariableSet (hy_env::mpi_node_id, new _Constant (hy_mpi_node_rank), false); + hy_env :: EnvVariableSet (hy_env::mpi_node_count, new _Constant (hy_mpi_node_count), false); +#endif + #if not defined (__HYPHY_MPI_MESSAGE_LOGGING__) && defined (__HYPHYMPI__) if (hy_mpi_node_rank == 0L) { #endif diff --git a/src/core/matrix.cpp b/src/core/matrix.cpp index 361be4e56..94fce9c88 100644 --- a/src/core/matrix.cpp +++ b/src/core/matrix.cpp @@ -6067,7 +6067,7 @@ HBLObjectRef _Matrix::Random (HBLObjectRef kind) { _String pdfkey ("PDF"), * arg0 = (_String *)keys->GetItem(0L); DeleteObject (keys); - if (arg0->Equal(&pdfkey)) { + if (arg0->Equal(pdfkey)) { _String pdf ((_String *) (pdfArgs->GetByKey(pdfkey,STRING))->toStr()), arg ("ARG0"); diff --git a/src/mains/unix.cpp b/src/mains/unix.cpp index 46a9ab3b0..8378c74e3 100644 --- a/src/mains/unix.cpp +++ b/src/mains/unix.cpp @@ -600,23 +600,16 @@ int main (int argc, char* argv[]) #endif #ifdef __HYPHYMPI__ - int rank, - size; - MPI_Init (&argc, &argv); + int rank, + size; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &size); - - hy_env :: EnvVariableSet (hy_env::mpi_node_id, new _Constant (rank), false); - hy_env :: EnvVariableSet (hy_env::mpi_node_count, new _Constant (size), false); - hy_mpi_node_rank = rank; hy_mpi_node_count = size; - - if (rank == 0) { - mpiNodesThatCantSwitch.Populate (size,1,0); - } - + + /* int i = 0; char hostname[256]; diff --git a/src/utils/hyphyunixutils.cpp b/src/utils/hyphyunixutils.cpp index afe3c1f24..8868e48e3 100644 --- a/src/utils/hyphyunixutils.cpp +++ b/src/utils/hyphyunixutils.cpp @@ -61,7 +61,6 @@ void mpiNormalLoop (int, int, _String &); void mpiOptimizerLoop (int, int); void mpiBgmLoop (int, int); -_SimpleList mpiNodesThatCantSwitch; #endif extern _List batchLanguageFunctionNames; @@ -242,7 +241,7 @@ void mpiNormalLoop (int rank, int size, _String & baseDir) ReportWarning ("[MPI] Returned from mpiOptimizer loop"); hyphyMPIOptimizerMode = _hyphyLFMPIModeNone; PushFilePath(baseDir, false, false); - } else if ( theMessage->Equal (&mpiLoopSwitchToBGM) ) { + } else if ( *theMessage == mpiLoopSwitchToBGM) { ReportWarning ("[MPI] Received signal to switch to mpiBgmLoop"); MPISendString (mpiLoopSwitchToBGM, senderID); // feedback to source to confirm receipt of message mpiBgmLoop (rank, size); @@ -291,8 +290,7 @@ void mpiNormalLoop (int rank, int size, _String & baseDir) } MPISendString(*resStr,senderID); - - + if (hy_env::EnvVariableTrue (preserveSlaveNodeState) == false) { PurgeAll (true); InitializeGlobals (); diff --git a/tests/hbltests/UnitTests/HBLCommands/Random.bf b/tests/hbltests/UnitTests/HBLCommands/Random.bf index 37f14f1bd..685ce1620 100644 --- a/tests/hbltests/UnitTests/HBLCommands/Random.bf +++ b/tests/hbltests/UnitTests/HBLCommands/Random.bf @@ -57,11 +57,11 @@ function runTest () { // *** set a breakpoint in malloc_error_break to debug // The code below returns the errors - //mean1 = {{1,1,1}}; - //cov1 = {{1,0}{0,1}}; - //a1 = {"PDF":"Gaussian","ARG0":cov1}; - //z = Random(mean1,a1); - //fprintf (stdout, 'z: ', z, '\n'); + mean1 = {{1,1,1}}; + cov1 = {{1,0}{0,1}}; + a1 = {"PDF":"Gaussian","ARG0":cov1}; + z = Random(mean1,a1); + fprintf (stdout, 'z: ', z, '\n'); //mean2 = {{1,1,1}}; //cov2 = {{1,0}{0,1}}; diff --git a/tests/hbltests/libv3/BUSTED.wbf b/tests/hbltests/libv3/BUSTED.wbf index 1dd70daa4..da09eebc9 100644 --- a/tests/hbltests/libv3/BUSTED.wbf +++ b/tests/hbltests/libv3/BUSTED.wbf @@ -2,5 +2,5 @@ LoadFunctionLibrary("SelectionAnalyses/BUSTED.bf", { "0": "Universal", "1": PATH_TO_CURRENT_BF + "data/CD2.nex", "2": "GROUP3", - "3": "All" + "3": "No" });