Skip to content

Commit

Permalink
Fix return type annotation of dataset size functions
Browse files Browse the repository at this point in the history
Fix the following mypy error (when using `strict_equality = True`):

```
lib/galaxy/datatypes/tabular.py:552: error: Non-overlapping equality check (left operand type: "int", right operand type: "str")  [comparison-overlap]
                            if dataset_fh.tell() != dataset.get_size():
```
  • Loading branch information
nsoranzo committed Mar 15, 2023
1 parent 1dc25d1 commit f45d307
Show file tree
Hide file tree
Showing 7 changed files with 22 additions and 14 deletions.
2 changes: 1 addition & 1 deletion lib/galaxy/datatypes/protocols.py
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ def get_converted_files_by_type(self, file_type):
def get_mime(self) -> str:
...

def get_size(self) -> str:
def get_size(self) -> int:
...

def has_data(self) -> bool:
Expand Down
18 changes: 14 additions & 4 deletions lib/galaxy/model/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
List,
NamedTuple,
Optional,
overload,
Set,
Tuple,
Type,
Expand Down Expand Up @@ -102,6 +103,7 @@
from sqlalchemy.orm.collections import attribute_mapped_collection
from sqlalchemy.sql import exists
from typing_extensions import (
Literal,
Protocol,
TypedDict,
)
Expand Down Expand Up @@ -3915,16 +3917,24 @@ def extra_files_path_name(self):
def _extra_files_rel_path(self):
return self._extra_files_path or self.extra_files_path_name

def _calculate_size(self):
def _calculate_size(self) -> int:
if self.external_filename:
try:
return os.path.getsize(self.external_filename)
except OSError:
return 0
else:
return self.object_store.size(self)
assert self.object_store
return self.object_store.size(self)

def get_size(self, nice_size=False, calculate_size=True):
@overload
def get_size(self, nice_size: Literal[False], calculate_size: bool = True) -> int:
...

@overload
def get_size(self, nice_size: Literal[True], calculate_size: bool = True) -> str:
...

def get_size(self, nice_size: bool = False, calculate_size: bool = True) -> Union[int, str]:
"""Returns the size of the data on disk"""
if self.file_size:
if nice_size:
Expand Down
4 changes: 2 additions & 2 deletions lib/galaxy/objectstore/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -170,7 +170,7 @@ def empty(self, obj, base_dir=None, extra_dir=None, extra_dir_at_root=False, alt
raise NotImplementedError()

@abc.abstractmethod
def size(self, obj, extra_dir=None, extra_dir_at_root=False, alt_name=None, obj_dir=False):
def size(self, obj, extra_dir=None, extra_dir_at_root=False, alt_name=None, obj_dir=False) -> int:
"""
Return size of the object identified by `obj`.
Expand Down Expand Up @@ -850,7 +850,7 @@ def _empty(self, obj, **kwargs):
"""Override `ObjectStore`'s stub by checking file size on disk."""
return self.size(obj, **kwargs) == 0

def _size(self, obj, **kwargs):
def _size(self, obj, **kwargs) -> int:
"""Override `ObjectStore`'s stub by return file size on disk.
Returns 0 if the object doesn't exist yet or other error.
Expand Down
3 changes: 1 addition & 2 deletions lib/galaxy/objectstore/cloud.py
Original file line number Diff line number Diff line change
Expand Up @@ -400,8 +400,7 @@ def _get_transfer_progress(self):
def _get_size_in_cloud(self, rel_path):
try:
obj = self.bucket.objects.get(rel_path)
if obj:
return obj.size
return obj.size
except Exception:
log.exception("Could not get size of key '%s' from S3", rel_path)
return -1
Expand Down
2 changes: 1 addition & 1 deletion lib/galaxy/objectstore/irods.py
Original file line number Diff line number Diff line change
Expand Up @@ -608,7 +608,7 @@ def _empty(self, obj, **kwargs):
else:
raise ObjectNotFound(f"objectstore.empty, object does not exist: {obj}, kwargs: {kwargs}")

def _size(self, obj, **kwargs):
def _size(self, obj, **kwargs) -> int:
ipt_timer = ExecutionTimer()
rel_path = self._construct_path(obj, **kwargs)
if self._in_cache(rel_path):
Expand Down
2 changes: 1 addition & 1 deletion lib/galaxy/objectstore/pithos.py
Original file line number Diff line number Diff line change
Expand Up @@ -309,7 +309,7 @@ def _empty(self, obj, **kwargs):
raise ObjectNotFound(f"objectstore.empty, object does not exist: {obj}, kwargs: {kwargs}")
return bool(self._size(obj, **kwargs))

def _size(self, obj, **kwargs):
def _size(self, obj, **kwargs) -> int:
"""
:returns: The size of the object, or 0 if it doesn't exist (sorry for
that, not our fault, the ObjectStore interface is like that some
Expand Down
5 changes: 2 additions & 3 deletions lib/galaxy/objectstore/s3.py
Original file line number Diff line number Diff line change
Expand Up @@ -385,9 +385,8 @@ def _get_transfer_progress(self):
def _get_size_in_s3(self, rel_path):
try:
key = self._bucket.get_key(rel_path)
if key:
return key.size
except S3ResponseError:
return key.size
except (S3ResponseError, AttributeError):
log.exception("Could not get size of key '%s' from S3", rel_path)
return -1

Expand Down

0 comments on commit f45d307

Please sign in to comment.