From f2b756b621e28ee5e5a4418e60bf928cf76681ec Mon Sep 17 00:00:00 2001 From: Lon Blauvelt Date: Thu, 2 Feb 2023 21:41:01 -0800 Subject: [PATCH 1/3] Update version_template.py --- version_template.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version_template.py b/version_template.py index a401ff0030..1821930d95 100644 --- a/version_template.py +++ b/version_template.py @@ -28,7 +28,7 @@ # - don't import even standard modules at global scope without renaming them # to have leading/trailing underscores -baseVersion = '5.9.0a1' +baseVersion = '5.10.0a1' cgcloudVersion = '1.6.0a1.dev393' From d6564095df2d31ff21e8ef36ed3c0d99107971b9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 2 Feb 2023 22:13:26 -0800 Subject: [PATCH 2/3] Update flake8-bugbear requirement from <21,>=20.11.1 to >=20.11.1,<24 (#4347) Updates the requirements on [flake8-bugbear](https://github.com/PyCQA/flake8-bugbear) to permit the latest version. - [Release notes](https://github.com/PyCQA/flake8-bugbear/releases) - [Commits](https://github.com/PyCQA/flake8-bugbear/compare/20.11.1...23.1.20) --- updated-dependencies: - dependency-name: flake8-bugbear dependency-type: direct:development ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 62f7e9e836..762d6146bb 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -12,7 +12,7 @@ types-setuptools types-boto types-pytz flake8>=3.8.4,<7 -flake8-bugbear>=20.11.1,<21 +flake8-bugbear>=20.11.1,<24 black isort pydocstyle From 92570421d671a94a5dfcb8c751aacefc63ff5b56 Mon Sep 17 00:00:00 2001 From: Lukas Ho <33379572+Hexotical@users.noreply.github.com> Date: Fri, 3 Feb 2023 19:28:19 -0800 Subject: [PATCH 3/3] Issues/4314 change build tag import (#4329) * Change where build tags dict is located to not require aws * Make typing proper * Update test imports * Move build_tag_dict to aws init * Import tag dict from the right place --------- Co-authored-by: Adam Novak --- src/toil/common.py | 4 +++- src/toil/jobStores/aws/jobStore.py | 4 ++-- src/toil/lib/aws/__init__.py | 21 +++++++++++++++++++++ src/toil/lib/aws/utils.py | 29 +++-------------------------- src/toil/test/lib/aws/test_utils.py | 2 +- src/toil/utils/toilLaunchCluster.py | 2 +- 6 files changed, 31 insertions(+), 31 deletions(-) diff --git a/src/toil/common.py b/src/toil/common.py index 728a4a8b9c..2f1c0723c6 100644 --- a/src/toil/common.py +++ b/src/toil/common.py @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import json import logging import os import pickle @@ -36,6 +37,7 @@ ContextManager, Dict, List, + MutableMapping, Optional, Set, Tuple, @@ -67,7 +69,7 @@ QueueSizeMessage, gen_message_bus_path) from toil.fileStores import FileID -from toil.lib.aws import zone_to_region +from toil.lib.aws import zone_to_region, build_tag_dict_from_env from toil.lib.compatibility import deprecated from toil.lib.conversions import bytes2human, human2bytes from toil.lib.io import try_path diff --git a/src/toil/jobStores/aws/jobStore.py b/src/toil/jobStores/aws/jobStore.py index 864437f336..b164120c69 100644 --- a/src/toil/jobStores/aws/jobStore.py +++ b/src/toil/jobStores/aws/jobStore.py @@ -35,6 +35,7 @@ from botocore.exceptions import ClientError import toil.lib.encryption as encryption +from toil.lib.aws import build_tag_dict_from_env from toil.fileStores import FileID from toil.jobStores.abstractJobStore import (AbstractJobStore, ConcurrentFileModificationException, @@ -56,8 +57,7 @@ ReadableTransformingPipe, WritablePipe) from toil.lib.aws.session import establish_boto3_session -from toil.lib.aws.utils import (build_tag_dict_from_env, - create_s3_bucket, +from toil.lib.aws.utils import (create_s3_bucket, flatten_tags, get_bucket_region, get_object_for_url, diff --git a/src/toil/lib/aws/__init__.py b/src/toil/lib/aws/__init__.py index e3a9e9549a..03c4584896 100644 --- a/src/toil/lib/aws/__init__.py +++ b/src/toil/lib/aws/__init__.py @@ -25,6 +25,7 @@ Dict, Iterable, List, + MutableMapping, Optional, TypeVar, Union) @@ -172,3 +173,23 @@ def running_on_ecs() -> bool: """ # We only care about relatively current ECS return 'ECS_CONTAINER_METADATA_URI_V4' in os.environ + +def build_tag_dict_from_env(environment: MutableMapping[str, str] = os.environ) -> Dict[str, str]: + tags = dict() + owner_tag = environment.get('TOIL_OWNER_TAG') + if owner_tag: + tags.update({'Owner': owner_tag}) + + user_tags = environment.get('TOIL_AWS_TAGS') + if user_tags: + try: + json_user_tags = json.loads(user_tags) + if isinstance(json_user_tags, dict): + tags.update(json.loads(user_tags)) + else: + logger.error('TOIL_AWS_TAGS must be in JSON format: {"key" : "value", ...}') + exit(1) + except json.decoder.JSONDecodeError: + logger.error('TOIL_AWS_TAGS must be in JSON format: {"key" : "value", ...}') + exit(1) + return tags diff --git a/src/toil/lib/aws/utils.py b/src/toil/lib/aws/utils.py index 2b0d89466e..38f44fabb7 100644 --- a/src/toil/lib/aws/utils.py +++ b/src/toil/lib/aws/utils.py @@ -28,7 +28,8 @@ Optional, Set, Union, - cast) + cast, + MutableMapping) from urllib.parse import ParseResult from toil.lib.aws import session @@ -41,10 +42,8 @@ retry) if sys.version_info >= (3, 8): - from typing import Literal, MutableMapping + from typing import Literal else: - from typing import MutableMapping - from typing_extensions import Literal try: @@ -403,28 +402,6 @@ def list_objects_for_url(url: ParseResult) -> List[str]: logger.debug('Found in %s items: %s', url, listing) return listing - -def build_tag_dict_from_env(environment: MutableMapping[str, str] = os.environ) -> Dict[str, str]: - tags = dict() - owner_tag = environment.get('TOIL_OWNER_TAG') - if owner_tag: - tags.update({'Owner': owner_tag}) - - user_tags = environment.get('TOIL_AWS_TAGS') - if user_tags: - try: - json_user_tags = json.loads(user_tags) - if isinstance(json_user_tags, dict): - tags.update(json.loads(user_tags)) - else: - logger.error('TOIL_AWS_TAGS must be in JSON format: {"key" : "value", ...}') - exit(1) - except json.decoder.JSONDecodeError: - logger.error('TOIL_AWS_TAGS must be in JSON format: {"key" : "value", ...}') - exit(1) - return tags - - def flatten_tags(tags: Dict[str, str]) -> List[Dict[str, str]]: """ Convert tags from a key to value dict into a list of 'Key': xxx, 'Value': xxx dicts. diff --git a/src/toil/test/lib/aws/test_utils.py b/src/toil/test/lib/aws/test_utils.py index 2f34d8a1d3..3de3862a94 100644 --- a/src/toil/test/lib/aws/test_utils.py +++ b/src/toil/test/lib/aws/test_utils.py @@ -18,7 +18,7 @@ import pytest -from toil.lib.aws.utils import build_tag_dict_from_env +from toil.lib.aws import build_tag_dict_from_env from toil.test import ToilTest logger = logging.getLogger(__name__) diff --git a/src/toil/utils/toilLaunchCluster.py b/src/toil/utils/toilLaunchCluster.py index 9e9a676e04..27bea8ad77 100644 --- a/src/toil/utils/toilLaunchCluster.py +++ b/src/toil/utils/toilLaunchCluster.py @@ -19,7 +19,7 @@ from toil import applianceSelf from toil.common import parser_with_common_options -from toil.lib.aws.utils import build_tag_dict_from_env +from toil.lib.aws import build_tag_dict_from_env from toil.provisioners import (check_valid_node_types, cluster_factory, parse_node_types)