Update with fix.

requirements-dev.txt

@@ -12,7 +12,7 @@ types-setuptools
 types-boto
 types-pytz
 flake8>=3.8.4,<7
-flake8-bugbear>=20.11.1,<21
+flake8-bugbear>=20.11.1,<24
 black
 isort
 pydocstyle

src/toil/common.py

@@ -11,6 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import json
 import logging
 import os
 import pickle
@@ -36,6 +37,7 @@
  ContextManager,
  Dict,
  List,
+ MutableMapping,
  Optional,
  Set,
  Tuple,
@@ -67,7 +69,7 @@
  QueueSizeMessage,
  gen_message_bus_path)
 from toil.fileStores import FileID
-from toil.lib.aws import zone_to_region
+from toil.lib.aws import zone_to_region, build_tag_dict_from_env
 from toil.lib.compatibility import deprecated
 from toil.lib.conversions import bytes2human, human2bytes
 from toil.lib.io import try_path

src/toil/jobStores/aws/jobStore.py

@@ -35,6 +35,7 @@
 from botocore.exceptions import ClientError
 
 import toil.lib.encryption as encryption
+from toil.lib.aws import build_tag_dict_from_env
 from toil.fileStores import FileID
 from toil.jobStores.abstractJobStore import (AbstractJobStore,
  ConcurrentFileModificationException,
@@ -56,8 +57,7 @@
  ReadableTransformingPipe,
  WritablePipe)
 from toil.lib.aws.session import establish_boto3_session
-from toil.lib.aws.utils import (build_tag_dict_from_env,
- create_s3_bucket,
+from toil.lib.aws.utils import (create_s3_bucket,
  flatten_tags,
  get_bucket_region,
  get_object_for_url,

src/toil/lib/aws/__init__.py

@@ -25,6 +25,7 @@
  Dict,
  Iterable,
  List,
+ MutableMapping,
  Optional,
  TypeVar,
  Union)
@@ -172,3 +173,23 @@ def running_on_ecs() -> bool:
  """
  # We only care about relatively current ECS
  return 'ECS_CONTAINER_METADATA_URI_V4' in os.environ
+
+def build_tag_dict_from_env(environment: MutableMapping[str, str] = os.environ) -> Dict[str, str]:
+ tags = dict()
+ owner_tag = environment.get('TOIL_OWNER_TAG')
+ if owner_tag:
+ tags.update({'Owner': owner_tag})
+
+ user_tags = environment.get('TOIL_AWS_TAGS')
+ if user_tags:
+ try:
+ json_user_tags = json.loads(user_tags)
+ if isinstance(json_user_tags, dict):
+ tags.update(json.loads(user_tags))
+ else:
+ logger.error('TOIL_AWS_TAGS must be in JSON format: {"key" : "value", ...}')
+ exit(1)
+ except json.decoder.JSONDecodeError:
+ logger.error('TOIL_AWS_TAGS must be in JSON format: {"key" : "value", ...}')
+ exit(1)
+ return tags

src/toil/lib/aws/utils.py

@@ -28,7 +28,8 @@
  Optional,
  Set,
  Union,
- cast)
+ cast,
+ MutableMapping)
 from urllib.parse import ParseResult
 
 from toil.lib.aws import session
@@ -41,10 +42,8 @@
  retry)
 
 if sys.version_info >= (3, 8):
- from typing import Literal, MutableMapping
+ from typing import Literal
 else:
- from typing import MutableMapping
-
  from typing_extensions import Literal
 
 try:
@@ -403,28 +402,6 @@ def list_objects_for_url(url: ParseResult) -> List[str]:
  logger.debug('Found in %s items: %s', url, listing)
  return listing
 
-
-def build_tag_dict_from_env(environment: MutableMapping[str, str] = os.environ) -> Dict[str, str]:
- tags = dict()
- owner_tag = environment.get('TOIL_OWNER_TAG')
- if owner_tag:
- tags.update({'Owner': owner_tag})
-
- user_tags = environment.get('TOIL_AWS_TAGS')
- if user_tags:
- try:
- json_user_tags = json.loads(user_tags)
- if isinstance(json_user_tags, dict):
- tags.update(json.loads(user_tags))
- else:
- logger.error('TOIL_AWS_TAGS must be in JSON format: {"key" : "value", ...}')
- exit(1)
- except json.decoder.JSONDecodeError:
- logger.error('TOIL_AWS_TAGS must be in JSON format: {"key" : "value", ...}')
- exit(1)
- return tags
-
-
 def flatten_tags(tags: Dict[str, str]) -> List[Dict[str, str]]:
  """
  Convert tags from a key to value dict into a list of 'Key': xxx, 'Value': xxx dicts.

src/toil/test/lib/aws/test_utils.py

@@ -18,7 +18,7 @@
 
 import pytest
 
-from toil.lib.aws.utils import build_tag_dict_from_env
+from toil.lib.aws import build_tag_dict_from_env
 from toil.test import ToilTest
 
 logger = logging.getLogger(__name__)

src/toil/utils/toilLaunchCluster.py

@@ -19,7 +19,7 @@
 
 from toil import applianceSelf
 from toil.common import parser_with_common_options
-from toil.lib.aws.utils import build_tag_dict_from_env
+from toil.lib.aws import build_tag_dict_from_env
 from toil.provisioners import (check_valid_node_types,
  cluster_factory,
  parse_node_types)

version_template.py

@@ -28,7 +28,7 @@
 # - don't import even standard modules at global scope without renaming them
 # to have leading/trailing underscores
 
-baseVersion = '5.9.1'
+baseVersion = '5.9.2'
 cgcloudVersion = '1.6.0a1.dev393'