From 7edc84fc5eeb88d9f9baead89b12858b96f5c2d5 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Fri, 12 May 2023 10:03:08 -0400 Subject: [PATCH 1/2] Refactor enum dependencies between schema and models. This is a piece that is needed to create a schema package - which I would really like to have around for #15639 and to in general write higher-level API tests. --- lib/galaxy/model/__init__.py | 58 ++++------------------- lib/galaxy/schema/schema.py | 92 ++++++++++++++++++++++++++++-------- 2 files changed, 81 insertions(+), 69 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index e1d1f5d4d889..78c7d0a6f597 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -129,6 +129,12 @@ from galaxy.model.orm.now import now from galaxy.model.orm.util import add_object_to_object_session from galaxy.objectstore import ObjectStore +from galaxy.schema.schema import ( + DatasetCollectionPopulatedState, + DatasetState, + DatasetValidatedState, + JobState, +) from galaxy.security import get_permitted_actions from galaxy.security.idencoding import IdEncodingHelper from galaxy.security.validate_user_input import validate_password_str @@ -1299,22 +1305,7 @@ class Job(Base, JobLike, UsesCreateAndUpdateTime, Dictifiable, Serializable): _numeric_metric = JobMetricNumeric _text_metric = JobMetricText - class states(str, Enum): - NEW = "new" - RESUBMITTED = "resubmitted" - UPLOAD = "upload" - WAITING = "waiting" - QUEUED = "queued" - RUNNING = "running" - OK = "ok" - ERROR = "error" - FAILED = "failed" - PAUSED = "paused" - DELETING = "deleting" - DELETED = "deleted" - STOPPING = "stop" - STOPPED = "stopped" - SKIPPED = "skipped" + states = JobState terminal_states = [states.OK, states.ERROR, states.DELETED] #: job states where the job hasn't finished and the model may still change @@ -3739,31 +3730,8 @@ class Dataset(Base, StorableObject, Serializable): back_populates="dataset", ) - class states(str, Enum): - NEW = "new" - UPLOAD = "upload" - QUEUED = "queued" - RUNNING = "running" - OK = "ok" - EMPTY = "empty" - ERROR = "error" - PAUSED = "paused" - SETTING_METADATA = "setting_metadata" - FAILED_METADATA = "failed_metadata" - # Non-deleted, non-purged datasets that don't have physical files. - # These shouldn't have objectstores attached - - # 'deferred' can be materialized for jobs using - # attached DatasetSource objects but 'discarded' - # cannot (e.g. imported histories). These should still - # be able to have history contents associated (normal HDAs?) - DEFERRED = "deferred" - DISCARDED = "discarded" - - @classmethod - def values(self): - return self.__members__.values() - # failed_metadata is only valid as DatasetInstance state currently + states = DatasetState non_ready_states = (states.NEW, states.UPLOAD, states.QUEUED, states.RUNNING, states.SETTING_METADATA) ready_states = tuple(set(states.__members__.values()) - set(non_ready_states)) @@ -4190,10 +4158,7 @@ class DatasetInstance(UsesCreateAndUpdateTime, _HasTable): purged: bool creating_job_associations: List[Union[JobToOutputDatasetCollectionAssociation, JobToOutputDatasetAssociation]] - class validated_states(str, Enum): - UNKNOWN = "unknown" - INVALID = "invalid" - OK = "ok" + validated_states = DatasetValidatedState def __init__( self, @@ -5963,10 +5928,7 @@ class DatasetCollection(Base, Dictifiable, UsesAnnotations, Serializable): dict_collection_visible_keys = ["id", "collection_type"] dict_element_visible_keys = ["id", "collection_type"] - class populated_states(str, Enum): - NEW = "new" # New dataset collection, unpopulated elements - OK = "ok" # Collection elements populated (HDAs may or may not have errors) - FAILED = "failed" # some problem populating state, won't be populated + populated_states = DatasetCollectionPopulatedState def __init__(self, id=None, collection_type=None, populated=True, element_count=None): self.id = id diff --git a/lib/galaxy/schema/schema.py b/lib/galaxy/schema/schema.py index b15f770bf4bc..a0def9736c4f 100644 --- a/lib/galaxy/schema/schema.py +++ b/lib/galaxy/schema/schema.py @@ -32,12 +32,6 @@ Literal, ) -from galaxy.model import ( - Dataset, - DatasetCollection, - DatasetInstance, - Job, -) from galaxy.schema.bco import XrefItem from galaxy.schema.fields import ( DecodedDatabaseIdField, @@ -64,6 +58,56 @@ OptionalNumberT = Optional[Union[int, float]] + +class DatasetState(str, Enum): + NEW = "new" + UPLOAD = "upload" + QUEUED = "queued" + RUNNING = "running" + OK = "ok" + EMPTY = "empty" + ERROR = "error" + PAUSED = "paused" + SETTING_METADATA = "setting_metadata" + FAILED_METADATA = "failed_metadata" + # Non-deleted, non-purged datasets that don't have physical files. + # These shouldn't have objectstores attached - + # 'deferred' can be materialized for jobs using + # attached DatasetSource objects but 'discarded' + # cannot (e.g. imported histories). These should still + # be able to have history contents associated (normal HDAs?) + DEFERRED = "deferred" + DISCARDED = "discarded" + + @classmethod + def values(self): + return self.__members__.values() + + +class JobState(str, Enum): + NEW = "new" + RESUBMITTED = "resubmitted" + UPLOAD = "upload" + WAITING = "waiting" + QUEUED = "queued" + RUNNING = "running" + OK = "ok" + ERROR = "error" + FAILED = "failed" + PAUSED = "paused" + DELETING = "deleting" + DELETED = "deleted" + STOPPING = "stop" + STOPPED = "stopped" + SKIPPED = "skipped" + + +class DatasetCollectionPopulatedState(str, Enum): + NEW = "new" # New dataset collection, unpopulated elements + OK = "ok" # Collection elements populated (HDAs may or may not have errors) + FAILED = "failed" # some problem populating state, won't be populated + + # Generic and common Field annotations that can be reused across models RelativeUrlField: RelativeUrl = Field( @@ -97,7 +141,7 @@ description="The encoded ID of this entity.", ) -DatasetStateField: Dataset.states = Field( +DatasetStateField: DatasetState = Field( ..., title="State", description="The current state of this dataset.", @@ -123,7 +167,7 @@ ), ) -PopulatedStateField: DatasetCollection.populated_states = Field( +PopulatedStateField: DatasetCollectionPopulatedState = Field( ..., title="Populated State", description=( @@ -442,7 +486,7 @@ class HDASummary(HistoryItemCommon): title="Dataset ID", description="The encoded ID of the dataset associated with this item.", ) - state: Dataset.states = DatasetStateField + state: DatasetState = DatasetStateField extension: str = Field( ..., title="Extension", @@ -460,7 +504,7 @@ class HDAInaccessible(HistoryItemBase): """History Dataset Association information when the user can not access it.""" accessible: bool = AccessibleField - state: Dataset.states = DatasetStateField + state: DatasetState = DatasetStateField HdaLddaField = Field( @@ -472,6 +516,12 @@ class HDAInaccessible(HistoryItemBase): ) +class DatasetValidatedState(str, Enum): + UNKNOWN = "unknown" + INVALID = "invalid" + OK = "ok" + + class HDADetailed(HDASummary): """History Dataset Association detailed information.""" @@ -572,7 +622,7 @@ class HDADetailed(HDASummary): title="Visualizations", description="The collection of visualizations that can be applied to this dataset.", ) - validated_state: DatasetInstance.validated_states = Field( + validated_state: DatasetValidatedState = Field( ..., title="Validated State", description="The state of the datatype validation for this dataset.", @@ -634,7 +684,7 @@ class DCSummary(Model): create_time: datetime = CreateTimeField update_time: datetime = UpdateTimeField collection_type: CollectionType = CollectionTypeField - populated_state: DatasetCollection.populated_states = PopulatedStateField + populated_state: DatasetCollectionPopulatedState = PopulatedStateField populated_state_message: Optional[str] = PopulatedStateMessageField element_count: Optional[int] = ElementCountField @@ -644,7 +694,7 @@ class HDAObject(Model): id: DecodedDatabaseIdField = EntityIdField model_class: HDA_MODEL_CLASS = ModelClassField(HDA_MODEL_CLASS) - state: Dataset.states = DatasetStateField + state: DatasetState = DatasetStateField hda_ldda: DatasetSourceType = HdaLddaField history_id: DecodedDatabaseIdField = HistoryIdField tags: List[str] @@ -789,7 +839,7 @@ class HDCASummary(HistoryItemCommon): ), ] = "collection" collection_type: CollectionType = CollectionTypeField - populated_state: DatasetCollection.populated_states = PopulatedStateField + populated_state: DatasetCollectionPopulatedState = PopulatedStateField populated_state_message: Optional[str] = PopulatedStateMessageField element_count: Optional[int] = ElementCountField job_source_id: Optional[DecodedDatabaseIdField] = Field( @@ -1003,8 +1053,8 @@ class HistoryActiveContentCounts(Model): ) -HistoryStateCounts = Dict[Dataset.states, int] -HistoryStateIds = Dict[Dataset.states, List[DecodedDatabaseIdField]] +HistoryStateCounts = Dict[DatasetState, int] +HistoryStateIds = Dict[DatasetState, List[DecodedDatabaseIdField]] class HistoryDetailed(HistorySummary): # Equivalent to 'dev-detailed' view, which seems the default @@ -1038,7 +1088,7 @@ class HistoryDetailed(HistorySummary): # Equivalent to 'dev-detailed' view, whi description="The relative URL in the form of /u/{username}/h/{slug}", ) genome_build: Optional[str] = GenomeBuildField - state: Dataset.states = Field( + state: DatasetState = Field( ..., title="State", description="The current state of the History based on the states of the datasets it contains.", @@ -1536,7 +1586,7 @@ class JobBaseModel(Model): title="History ID", description="The encoded ID of the history associated with this item.", ) - state: Job.states = Field( + state: JobState = Field( ..., title="State", description="Current state of the job.", @@ -1566,8 +1616,8 @@ class JobImportHistoryResponse(JobBaseModel): class ItemStateSummary(Model): id: DecodedDatabaseIdField = EntityIdField - populated_state: DatasetCollection.populated_states = PopulatedStateField - states: Dict[Job.states, int] = Field( + populated_state: DatasetCollectionPopulatedState = PopulatedStateField + states: Dict[JobState, int] = Field( {}, title="States", description=("A dictionary of job states and the number of jobs in that state.") ) @@ -2770,7 +2820,7 @@ class FileLibraryFolderItem(LibraryFolderItemBase): date_uploaded: datetime is_unrestricted: bool is_private: bool - state: Dataset.states = DatasetStateField + state: DatasetState = DatasetStateField file_size: str raw_size: int ldda_id: EncodedDatabaseIdField From c040d83665d9a8c61bad886dcab3fea91f5403b6 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Fri, 12 May 2023 10:07:10 -0400 Subject: [PATCH 2/2] Ooo... look it cleaned up schema.ts nicely also! --- client/src/schema/schema.ts | 126 ++++++++++++++++++------------------ 1 file changed, 63 insertions(+), 63 deletions(-) diff --git a/client/src/schema/schema.ts b/client/src/schema/schema.ts index a53f9ea37327..034563cbe54f 100644 --- a/client/src/schema/schema.ts +++ b/client/src/schema/schema.ts @@ -2472,6 +2472,12 @@ export interface components { * @description Represents a collection of elements contained in the dataset collection. */ DatasetCollectionContentElements: components["schemas"]["DCESummary"][]; + /** + * DatasetCollectionPopulatedState + * @description An enumeration. + * @enum {string} + */ + DatasetCollectionPopulatedState: "new" | "ok" | "failed"; /** * DatasetContentType * @description For retrieving content from a structured dataset (e.g. HDF5) @@ -2557,6 +2563,24 @@ export interface components { * @enum {string} */ DatasetSourceType: "hda" | "ldda"; + /** + * DatasetState + * @description An enumeration. + * @enum {string} + */ + DatasetState: + | "new" + | "upload" + | "queued" + | "running" + | "ok" + | "empty" + | "error" + | "paused" + | "setting_metadata" + | "failed_metadata" + | "deferred" + | "discarded"; /** * DatasetStorageDetails * @description Base model definition with common configuration used by all derived models. @@ -2634,6 +2658,12 @@ export interface components { */ truncated: boolean; }; + /** + * DatasetValidatedState + * @description An enumeration. + * @enum {string} + */ + DatasetValidatedState: "unknown" | "invalid" | "ok"; /** DatatypeConverter */ DatatypeConverter: { /** @@ -3178,7 +3208,7 @@ export interface components { * State * @description The current state of this dataset. */ - state: components["schemas"]["galaxy__model__Dataset__states"]; + state: components["schemas"]["DatasetState"]; /** Tags */ tags: string; /** @@ -3684,7 +3714,7 @@ export interface components { * State * @description The current state of this dataset. */ - state: components["schemas"]["galaxy__model__Dataset__states"]; + state: components["schemas"]["DatasetState"]; tags: components["schemas"]["TagCollection"]; /** * Type @@ -3721,7 +3751,7 @@ export interface components { * Validated State * @description The state of the datatype validation for this dataset. */ - validated_state: components["schemas"]["validated_states"]; + validated_state: components["schemas"]["DatasetValidatedState"]; /** * Validated State Message * @description The message with details about the datatype validation result for this dataset. @@ -3772,7 +3802,7 @@ export interface components { * State * @description The current state of this dataset. */ - state: components["schemas"]["galaxy__model__Dataset__states"]; + state: components["schemas"]["DatasetState"]; /** Tags */ tags: string[]; }; @@ -3840,7 +3870,7 @@ export interface components { * State * @description The current state of this dataset. */ - state: components["schemas"]["galaxy__model__Dataset__states"]; + state: components["schemas"]["DatasetState"]; tags: components["schemas"]["TagCollection"]; /** * Type @@ -3978,7 +4008,7 @@ export interface components { * Populated State * @description Indicates the general state of the elements in the dataset collection:- 'new': new dataset collection, unpopulated elements.- 'ok': collection elements populated (HDAs may or may not have errors).- 'failed': some problem populating, won't be populated. */ - populated_state: components["schemas"]["populated_states"]; + populated_state: components["schemas"]["DatasetCollectionPopulatedState"]; /** * Populated State Message * @description Optional message with further information in case the population of the dataset collection failed. @@ -4107,7 +4137,7 @@ export interface components { * Populated State * @description Indicates the general state of the elements in the dataset collection:- 'new': new dataset collection, unpopulated elements.- 'ok': collection elements populated (HDAs may or may not have errors).- 'failed': some problem populating, won't be populated. */ - populated_state: components["schemas"]["populated_states"]; + populated_state: components["schemas"]["DatasetCollectionPopulatedState"]; /** * Populated State Message * @description Optional message with further information in case the population of the dataset collection failed. @@ -4531,7 +4561,7 @@ export interface components { * State * @description The current state of the History based on the states of the datasets it contains. */ - state: components["schemas"]["galaxy__model__Dataset__states"]; + state: components["schemas"]["DatasetState"]; /** * State Counts * @description A dictionary keyed to possible dataset states and valued with the number of datasets in this history that have those states. @@ -4683,7 +4713,7 @@ export interface components { * Populated State * @description Indicates the general state of the elements in the dataset collection:- 'new': new dataset collection, unpopulated elements.- 'ok': collection elements populated (HDAs may or may not have errors).- 'failed': some problem populating, won't be populated. */ - populated_state: components["schemas"]["populated_states"]; + populated_state: components["schemas"]["DatasetCollectionPopulatedState"]; /** * States * @description A dictionary of job states and the number of jobs in that state. @@ -4964,7 +4994,7 @@ export interface components { * State * @description Current state of the job. */ - state: components["schemas"]["galaxy__model__Job__states"]; + state: components["schemas"]["JobState"]; /** * Tool ID * @description Identifier of the tool that generated this job. @@ -5003,6 +5033,27 @@ export interface components { * @enum {string} */ JobSourceType: "Job" | "ImplicitCollectionJobs" | "WorkflowInvocation"; + /** + * JobState + * @description An enumeration. + * @enum {string} + */ + JobState: + | "new" + | "resubmitted" + | "upload" + | "waiting" + | "queued" + | "running" + | "ok" + | "error" + | "failed" + | "paused" + | "deleting" + | "deleted" + | "stop" + | "stopped" + | "skipped"; /** * JobStateSummary * @description Base model definition with common configuration used by all derived models. @@ -5025,7 +5076,7 @@ export interface components { * Populated State * @description Indicates the general state of the elements in the dataset collection:- 'new': new dataset collection, unpopulated elements.- 'ok': collection elements populated (HDAs may or may not have errors).- 'failed': some problem populating, won't be populated. */ - populated_state: components["schemas"]["populated_states"]; + populated_state: components["schemas"]["DatasetCollectionPopulatedState"]; /** * States * @description A dictionary of job states and the number of jobs in that state. @@ -7475,7 +7526,7 @@ export interface components { * Populated State * @description Indicates the general state of the elements in the dataset collection:- 'new': new dataset collection, unpopulated elements.- 'ok': collection elements populated (HDAs may or may not have errors).- 'failed': some problem populating, won't be populated. */ - populated_state: components["schemas"]["populated_states"]; + populated_state: components["schemas"]["DatasetCollectionPopulatedState"]; /** * States * @description A dictionary of job states and the number of jobs in that state. @@ -7615,57 +7666,6 @@ export interface components { */ namespace: string; }; - /** - * states - * @description An enumeration. - * @enum {string} - */ - galaxy__model__Dataset__states: - | "new" - | "upload" - | "queued" - | "running" - | "ok" - | "empty" - | "error" - | "paused" - | "setting_metadata" - | "failed_metadata" - | "deferred" - | "discarded"; - /** - * states - * @description An enumeration. - * @enum {string} - */ - galaxy__model__Job__states: - | "new" - | "resubmitted" - | "upload" - | "waiting" - | "queued" - | "running" - | "ok" - | "error" - | "failed" - | "paused" - | "deleting" - | "deleted" - | "stop" - | "stopped" - | "skipped"; - /** - * populated_states - * @description An enumeration. - * @enum {string} - */ - populated_states: "new" | "ok" | "failed"; - /** - * validated_states - * @description An enumeration. - * @enum {string} - */ - validated_states: "unknown" | "invalid" | "ok"; }; responses: never; parameters: never;