Skip to content

Commit

Permalink
Fix mypy sqla errors around Dataset.transform.
Browse files Browse the repository at this point in the history
  • Loading branch information
jmchilton committed Apr 4, 2024
1 parent 8fe5d64 commit 41f0572
Show file tree
Hide file tree
Showing 2 changed files with 15 additions and 5 deletions.
14 changes: 12 additions & 2 deletions lib/galaxy/model/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -208,8 +208,18 @@

OPTIONAL_STR_TO_STR_DICT = Optional[Dict[str, str]]


class TransformAction(TypedDict):
action: str


TRANSFORM_ACTIONS = Optional[List[TransformAction]]

mapper_registry = registry(
type_annotation_map={OPTIONAL_STR_TO_STR_DICT: JSONType},
type_annotation_map={
OPTIONAL_STR_TO_STR_DICT: JSONType,
TRANSFORM_ACTIONS: MutableJSONType,
},
)

# When constructing filters with in for a fixed set of ids, maximum
Expand Down Expand Up @@ -4364,7 +4374,7 @@ class DatasetSource(Base, Dictifiable, Serializable):
dataset_id: Mapped[Optional[int]] = mapped_column(Integer, ForeignKey("dataset.id"), index=True)
source_uri: Mapped[Optional[str]] = mapped_column(TEXT)
extra_files_path: Mapped[Optional[str]] = mapped_column(TEXT)
transform: Mapped[Optional[bytes]] = mapped_column(MutableJSONType)
transform: Mapped[TRANSFORM_ACTIONS] = mapped_column(MutableJSONType)
dataset = relationship("Dataset", back_populates="sources")
hashes = relationship("DatasetSourceHash", back_populates="source")
dict_collection_visible_keys = ["id", "source_uri", "extra_files_path", "transform"]
Expand Down
6 changes: 3 additions & 3 deletions lib/galaxy/model/deferred.py
Original file line number Diff line number Diff line change
Expand Up @@ -178,20 +178,20 @@ def ensure_materialized(

def _stream_source(self, target_source: DatasetSource, datatype) -> str:
path = stream_url_to_file(target_source.source_uri, file_sources=self._file_sources)
transform = target_source.transform or [] # type:ignore[var-annotated]
transform = target_source.transform or []
to_posix_lines = False
spaces_to_tabs = False
datatype_groom = False
for transform_action in transform:
action = transform_action["action"] # type:ignore[index]
action = transform_action["action"]
if action == "to_posix_lines":
to_posix_lines = True
elif action == "spaces_to_tabs":
spaces_to_tabs = True
elif action == "datatype_groom":
datatype_groom = True
else:
raise Exception(f"Failed to materialize dataest, unknown transformation action {action} applied.")
raise Exception(f"Failed to materialize dataset, unknown transformation action {action} applied.")
if to_posix_lines or spaces_to_tabs:
convert_fxn = convert_function(to_posix_lines, spaces_to_tabs)
convert_result = convert_fxn(path, False)
Expand Down

0 comments on commit 41f0572

Please sign in to comment.