From 99e36a7a26fb84f63faa26059ecab4f8f1449995 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Thu, 17 Nov 2022 21:27:43 -0500 Subject: [PATCH 1/2] Add resource_params to context --- tpv/core/mapper.py | 7 ++++--- tpv/rules/gateway.py | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/tpv/core/mapper.py b/tpv/core/mapper.py index 01bf44a..bc8797a 100644 --- a/tpv/core/mapper.py +++ b/tpv/core/mapper.py @@ -93,7 +93,7 @@ def _find_matching_entities(self, tool, user): return entity_list - def match_combine_evaluate_entities(self, app, tool, user, job): + def match_combine_evaluate_entities(self, app, tool, user, job, resource_params): # 1. Find the entities relevant to this job entity_list = self._find_matching_entities(tool, user) @@ -105,6 +105,7 @@ def match_combine_evaluate_entities(self, app, tool, user, job): 'tool': tool, 'user': user, 'job': job, + 'resource_params': resource_params, 'mapper': self }) @@ -128,9 +129,9 @@ def match_combine_evaluate_entities(self, app, tool, user, job): return context, evaluated_entity - def map_to_destination(self, app, tool, user, job): + def map_to_destination(self, app, tool, user, job, resource_params): # 1. Find, combine and evaluate entities that match this tool and user - context, evaluated_entity = self.match_combine_evaluate_entities(app, tool, user, job) + context, evaluated_entity = self.match_combine_evaluate_entities(app, tool, user, job, resource_params) # 2. Shortlist destinations with tags that match the combined entity ranked_dest_entities = self.find_matching_destinations(evaluated_entity, self.destinations, context) diff --git a/tpv/rules/gateway.py b/tpv/rules/gateway.py index f2b6056..169fb7a 100644 --- a/tpv/rules/gateway.py +++ b/tpv/rules/gateway.py @@ -43,8 +43,8 @@ def reload_destination_mapper(path=None): return mapper -def map_tool_to_destination(app, job, tool, user, tpv_config_files): +def map_tool_to_destination(app, job, tool, user, resource_params, tpv_config_files): global ACTIVE_DESTINATION_MAPPER if not ACTIVE_DESTINATION_MAPPER: ACTIVE_DESTINATION_MAPPER = setup_destination_mapper(app, tpv_config_files) - return ACTIVE_DESTINATION_MAPPER.map_to_destination(app, tool, user, job) + return ACTIVE_DESTINATION_MAPPER.map_to_destination(app, tool, user, job, resource_params) From 873d02dec12b27e2f799e952faab2c080e582d25 Mon Sep 17 00:00:00 2001 From: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com> Date: Thu, 8 Dec 2022 12:01:07 +0530 Subject: [PATCH 2/2] Add more context parameters + some minor refactoring --- tests/test_entity.py | 6 +++++- tpv/core/mapper.py | 51 ++++++++++++++++++++++++-------------------- tpv/rules/gateway.py | 6 ++++-- 3 files changed, 37 insertions(+), 26 deletions(-) diff --git a/tests/test_entity.py b/tests/test_entity.py index 7ce3729..19784ba 100644 --- a/tests/test_entity.py +++ b/tests/test_entity.py @@ -28,8 +28,12 @@ def test_all_entities_refer_to_same_loader(self): # get the original loader original_loader = gateway.ACTIVE_DESTINATION_MAPPER.loader + context = { + 'app': app, + 'job': job + } # make sure we are still referring to the same loader after evaluation - _, evaluated_entity = gateway.ACTIVE_DESTINATION_MAPPER.match_combine_evaluate_entities(app, tool, user, job) + evaluated_entity = gateway.ACTIVE_DESTINATION_MAPPER.match_combine_evaluate_entities(context, tool, user) assert evaluated_entity.loader == original_loader for rule in evaluated_entity.rules: assert rule.loader == original_loader diff --git a/tpv/core/mapper.py b/tpv/core/mapper.py index bc8797a..fabc4c4 100644 --- a/tpv/core/mapper.py +++ b/tpv/core/mapper.py @@ -59,7 +59,7 @@ def combine_entities(self, entities): def rank(self, entity, destinations, context): return entity.rank_destinations(destinations, context) - def find_matching_destinations(self, entity, destinations, context): + def match_and_rank_destinations(self, entity, destinations, context): matches = [dest for dest in destinations.values() if dest.matches(entity, context)] return self.rank(entity, matches, context) @@ -93,30 +93,18 @@ def _find_matching_entities(self, tool, user): return entity_list - def match_combine_evaluate_entities(self, app, tool, user, job, resource_params): + def match_combine_evaluate_entities(self, context, tool, user): # 1. Find the entities relevant to this job entity_list = self._find_matching_entities(tool, user) - # 2. Create evaluation context - these are the common variables available within any code block - context = {} - context.update(self.global_context or {}) - context.update({ - 'app': app, - 'tool': tool, - 'user': user, - 'job': job, - 'resource_params': resource_params, - 'mapper': self - }) - - # 3. Combine entity requirements + # 2. Combine entity requirements combined_entity = self.combine_entities(entity_list) context.update({ 'entity': combined_entity, 'self': combined_entity }) - # 4. Evaluate expressions + # 3. Evaluate expressions evaluated_entity = combined_entity.evaluate(context) context.update({ 'entity': evaluated_entity, @@ -127,16 +115,32 @@ def match_combine_evaluate_entities(self, app, tool, user, job, resource_params) # with destinations evaluated_entity.rules = {} - return context, evaluated_entity + return evaluated_entity + + def map_to_destination(self, app, tool, user, job, job_wrapper=None, resource_params=None, + workflow_invocation_uuid=None): + + # 1. Create evaluation context - these are the common variables available within any code block + context = {} + context.update(self.global_context or {}) + context.update({ + 'app': app, + 'tool': tool, + 'user': user, + 'job': job, + 'job_wrapper': job_wrapper, + 'resource_params': resource_params, + 'workflow_invocation_uuid': workflow_invocation_uuid, + 'mapper': self + }) - def map_to_destination(self, app, tool, user, job, resource_params): - # 1. Find, combine and evaluate entities that match this tool and user - context, evaluated_entity = self.match_combine_evaluate_entities(app, tool, user, job, resource_params) + # 2. Find, combine and evaluate entities that match this tool and user + evaluated_entity = self.match_combine_evaluate_entities(context, tool, user) - # 2. Shortlist destinations with tags that match the combined entity - ranked_dest_entities = self.find_matching_destinations(evaluated_entity, self.destinations, context) + # 3. Match and rank destinations that best match the combined entity + ranked_dest_entities = self.match_and_rank_destinations(evaluated_entity, self.destinations, context) - # 3. Fully combine entity with matching destinations + # 4. Fully combine entity with matching destinations if ranked_dest_entities: wait_exception_raised = False for d in ranked_dest_entities: @@ -146,6 +150,7 @@ def map_to_destination(self, app, tool, user, job, resource_params): gxy_destination = app.job_config.get_destination(d.id) if evaluated_destination.params.get('destination_name_override'): gxy_destination.id = evaluated_destination.params.get('destination_name_override') + # 5. Return the top-ranked destination that evaluates successfully return self.configure_gxy_destination(gxy_destination, evaluated_destination) except TryNextDestinationOrFail as ef: log.exception(f"Destination entity: {d} matched but could not fulfill requirements due to: {ef}." diff --git a/tpv/rules/gateway.py b/tpv/rules/gateway.py index 169fb7a..f06a074 100644 --- a/tpv/rules/gateway.py +++ b/tpv/rules/gateway.py @@ -43,8 +43,10 @@ def reload_destination_mapper(path=None): return mapper -def map_tool_to_destination(app, job, tool, user, resource_params, tpv_config_files): +def map_tool_to_destination(app, job, tool, user, tpv_config_files, job_wrapper=None, resource_params=None, + workflow_invocation_uuid=None): global ACTIVE_DESTINATION_MAPPER if not ACTIVE_DESTINATION_MAPPER: ACTIVE_DESTINATION_MAPPER = setup_destination_mapper(app, tpv_config_files) - return ACTIVE_DESTINATION_MAPPER.map_to_destination(app, tool, user, job, resource_params) + return ACTIVE_DESTINATION_MAPPER.map_to_destination(app, tool, user, job, job_wrapper, resource_params, + workflow_invocation_uuid)