Merge branch 'release_21.05' into dev

.github/workflows/integration_selenium.yaml

@@ -5,8 +5,6 @@ concurrency:
   cancel-in-progress: true
 env:
   GALAXY_TEST_DBURI: 'postgresql://postgres:postgres@localhost:5432/galaxy?client_encoding=utf8'
-  GALAXY_TEST_SELENIUM_REMOTE: '1'
-  GALAXY_TEST_SELENIUM_REMOTE_PORT: "4444"
   GALAXY_SKIP_CLIENT_BUILD: '0'
   GALAXY_TEST_SELENIUM_RETRIES: 1
   YARN_INSTALL_OPTS: --frozen-lockfile
@@ -26,10 +24,6 @@ jobs:
           POSTGRES_DB: postgres
         ports:
           - 5432:5432
-      selenium:
-        image: selenium/standalone-chrome:3.141.59
-        ports:
-          - 4444:4444
     steps:
       - name: Prune unused docker image, volumes and containers
         run: docker system prune -a -f
@@ -48,6 +42,7 @@ jobs:
       - uses: mvdbeek/gha-yarn-cache@master
         with:
           yarn-lock-file: 'galaxy root/client/yarn.lock'
+      - uses: nanasess/setup-chromedriver@master
       - name: Run tests
         run: './run_tests.sh -integration test/integration_selenium'
         working-directory: 'galaxy root'

client/src/components/User/UserPreferencesModel.js

@@ -1,15 +1,16 @@
 import { getGalaxyInstance } from "app";
 import _l from "utils/localization";
 
-export const getUserPreferencesModel = () => {
+export const getUserPreferencesModel = (user_id) => {
     const Galaxy = getGalaxyInstance();
     const config = Galaxy.config;
+    user_id = user_id || Galaxy.user.id;
     return {
         information: {
             title: _l("Manage Information"),
             id: "edit-preferences-information",
             description: "Edit your email, addresses and custom parameters or change your public name.",
-            url: `api/users/${Galaxy.user.id}/information/inputs`,
+            url: `api/users/${user_id}/information/inputs`,
             icon: "fa-user",
             redirect: "user",
             shouldRender: !config.use_remote_user && config.enable_account_interface,
@@ -19,7 +20,7 @@ export const getUserPreferencesModel = () => {
             id: "edit-preferences-password",
             description: _l("Allows you to change your login credentials."),
             icon: "fa-unlock-alt",
-            url: `api/users/${Galaxy.user.id}/password/inputs`,
+            url: `api/users/${user_id}/password/inputs`,
             submit_title: "Save Password",
             redirect: "user",
             shouldRender: !config.use_remote_user && config.enable_account_interface,
@@ -38,7 +39,7 @@ export const getUserPreferencesModel = () => {
             id: "edit-preferences-permissions",
             description:
                 "Grant others default access to newly created histories. Changes made here will only affect histories created after these settings have been stored.",
-            url: `api/users/${Galaxy.user.id}/permissions/inputs`,
+            url: `api/users/${user_id}/permissions/inputs`,
             icon: "fa-users",
             submit_title: "Save Permissions",
             redirect: "user",
@@ -55,7 +56,7 @@ export const getUserPreferencesModel = () => {
             title: _l("Manage API Key"),
             id: "edit-preferences-api-key",
             description: _l("Access your current API key or create a new one."),
-            url: `api/users/${Galaxy.user.id}/api_key/inputs`,
+            url: `api/users/${user_id}/api_key/inputs`,
             icon: "fa-key",
             submit_title: "Create a new Key",
             submit_icon: "fa-check",
@@ -73,7 +74,7 @@ export const getUserPreferencesModel = () => {
             title: _l("Manage Toolbox Filters"),
             id: "edit-preferences-toolbox-filters",
             description: _l("Customize your Toolbox by displaying or omitting sets of Tools."),
-            url: `api/users/${Galaxy.user.id}/toolbox_filters/inputs`,
+            url: `api/users/${user_id}/toolbox_filters/inputs`,
             icon: "fa-filter",
             submit_title: "Save Filters",
             redirect: "user",

client/src/entry/analysis/AnalysisRouter.js

@@ -138,10 +138,8 @@ export const getAnalysisRouter = (Galaxy) => {
             });
         },
 
-        show_user_form: function (form_id) {
-            const Galaxy = getGalaxyInstance();
-            const model = getUserPreferencesModel();
-            model.user_id = Galaxy.params.id;
+        show_user_form: function (form_id, params) {
+            const model = getUserPreferencesModel(params.id);
             this.page.display(new FormWrapper.View(_.extend(model[form_id], { active_tab: "user" })));
         },
 

lib/galaxy/jobs/handler.py

@@ -21,6 +21,7 @@
 )
 
 from galaxy import model
+from galaxy.exceptions import ObjectNotFound
 from galaxy.jobs import (
     JobDestination,
     JobWrapper,
@@ -1087,6 +1088,10 @@ def recover(self, job, job_wrapper):
         except KeyError:
             log.error(f'recover(): ({job_wrapper.job_id}) Invalid job runner: {runner_name}')
             job_wrapper.fail(DEFAULT_JOB_PUT_FAILURE_MESSAGE)
+        except ObjectNotFound:
+            msg = "Could not recover job working directory after Galaxy restart"
+            log.exception(f"recover(): ({job_wrapper.job_id}) {msg}")
+            job_wrapper.fail(msg)
 
     def shutdown(self):
         failures = []

lib/galaxy/model/__init__.py

@@ -4251,6 +4251,7 @@ def _get_nested_collection_attributes(
         dce = alias(DatasetCollectionElement)
 
         depth_collection_type = dataset_collection.collection_type
+        order_by_columns = [dce.c.element_index]
         nesting_level = 0
 
         def attribute_columns(column_collection, attributes, nesting_level=None):
@@ -4269,6 +4270,7 @@ def attribute_columns(column_collection, attributes, nesting_level=None):
             nesting_level += 1
             inner_dc = alias(DatasetCollection)
             inner_dce = alias(DatasetCollectionElement)
+            order_by_columns.append(inner_dce.c.element_index)
             q = q.join(
                 inner_dc, inner_dc.c.id == dce.c.child_collection_id
             ).join(
@@ -4297,7 +4299,7 @@ def attribute_columns(column_collection, attributes, nesting_level=None):
             q = q.add_entity(entity)
             if entity == DatasetCollectionElement:
                 q = q.filter(entity.id == dce.c.id)
-        return q.distinct()
+        return q.distinct(*order_by_columns).order_by(*order_by_columns)
 
     @property
     def dataset_states_and_extensions_summary(self):

scripts/cleanup_datasets/pgcleanup.py

@@ -12,6 +12,7 @@
 import os
 import string
 import sys
+import time
 from collections import namedtuple
 from functools import partial
 
@@ -105,6 +106,7 @@ def __init__(self, app):
         self._debug = app.args.debug
         self._update_time = app.args.update_time
         self._force_retry = app.args.force_retry
+        self._epoch_time = str(int(time.time()))
         self._days = app.args.days
         self._config = app.config
         self._update = app._update
@@ -141,6 +143,7 @@ def __open_log(self):
         self.__log.propagate = False
         m = ('==== Log opened: %s ' % datetime.datetime.now().isoformat()).ljust(72, '=')
         self.__log.info(m)
+        self.__log.info(f'Epoch time for this action: {self._epoch_time}')
 
     def __close_log(self):
         m = ('==== Log closed: %s ' % datetime.datetime.now().isoformat()).ljust(72, '=')
@@ -188,6 +191,7 @@ def sql(self):
         return self._action_sql.format(
             update_time_sql=self._update_time_sql,
             force_retry_sql=self._force_retry_sql,
+            epoch_time=self._epoch_time,
         )
 
     @property
@@ -334,11 +338,13 @@ def sql(self):
         _purge_hda_dependencies_sql = self._purge_hda_dependencies_sql.format(
             update_time_sql=self._update_time_sql,
             force_retry_sql=self._force_retry_sql,
+            epoch_time=self._epoch_time,
         )
         return self._action_sql.format(
             purge_hda_dependencies_sql=_purge_hda_dependencies_sql,
             update_time_sql=self._update_time_sql,
             force_retry_sql=self._force_retry_sql,
+            epoch_time=self._epoch_time,
         )
 
 
@@ -525,6 +531,7 @@ class DeleteInactiveUsers(Action):
 class PurgeDeletedUsers(PurgesHDAs, RemovesMetadataFiles, Action):
     """
     - Mark purged all users that are older than the specified number of days.
+    - Set User.disk_usage = 0 for all users purged in this step.
     - Mark purged all Histories whose user_ids are purged in this step.
     - Mark purged all HistoryDatasetAssociations whose history_ids are purged in this step.
     - Delete all UserGroupAssociations whose user_ids are purged in this step.
@@ -655,6 +662,121 @@ def zero_disk_usage(self):
         self.log.info('zero_disk_usage user_ids: %s', ' '.join(str(i) for i in user_ids))
 
 
+class PurgeDeletedUsersGDPR(PurgesHDAs, RemovesMetadataFiles, Action):
+    """
+    - Perform all steps in the PurgeDeletedUsers/purge_deleted_users action
+    - Obfuscate User.email and User.username for all users purged in this step.
+
+    NOTE: Your database must have the pgcrypto extension installed e.g. with:
+      CREATE EXTENSION IF NOT EXISTS pgcrypto;
+    """
+    _action_sql = """
+        WITH purged_user_ids
+          AS (     UPDATE galaxy_user
+                      SET purged = true,
+                          disk_usage = 0,
+                          email = encode(digest(email || '{epoch_time}', 'sha1'), 'hex'),
+                          username = encode(digest(username || '{epoch_time}', 'sha1'), 'hex'){update_time_sql}
+                    WHERE deleted{force_retry_sql}
+                          AND update_time < (NOW() AT TIME ZONE 'utc' - interval '%(days)s days')
+                RETURNING id),
+             deleted_uga_ids
+          AS (DELETE FROM user_group_association
+                    USING purged_user_ids
+                    WHERE user_group_association.user_id = purged_user_ids.id
+                RETURNING user_group_association.user_id AS user_id,
+                          user_group_association.id AS id),
+             deleted_ura_ids
+          AS (DELETE FROM user_role_association
+                    USING role
+                    WHERE role.id = user_role_association.role_id
+                          AND role.type != 'private'
+                          AND user_role_association.user_id IN
+                            (SELECT id
+                               FROM purged_user_ids)
+                RETURNING user_role_association.user_id AS user_id,
+                          user_role_association.id AS id),
+             deleted_ua_ids
+          AS (DELETE FROM user_address
+                    USING purged_user_ids
+                    WHERE user_address.user_id = purged_user_ids.id
+                RETURNING user_address.user_id AS user_id,
+                          user_address.id AS id),
+             user_events
+          AS (INSERT INTO cleanup_event_user_association
+                          (create_time, cleanup_event_id, user_id)
+                   SELECT NOW() AT TIME ZONE 'utc', %(event_id)s, id
+                     FROM purged_user_ids),
+             purged_history_ids
+          AS (     UPDATE history
+                      SET purged = true{update_time_sql}
+                     FROM purged_user_ids
+                    WHERE purged_user_ids.id = history.user_id
+                          AND NOT history.purged
+                RETURNING history.user_id AS user_id,
+                          history.id AS id),
+             history_events
+          AS (INSERT INTO cleanup_event_history_association
+                          (create_time, cleanup_event_id, history_id)
+                   SELECT NOW() AT TIME ZONE 'utc', %(event_id)s, id
+                     FROM purged_history_ids),
+             purged_hda_ids
+          AS (     UPDATE history_dataset_association
+                      SET purged = true, deleted = true{update_time_sql}
+                     FROM purged_history_ids
+                    WHERE purged_history_ids.id = history_dataset_association.history_id
+                          AND NOT history_dataset_association.purged
+                RETURNING history_dataset_association.history_id AS history_id,
+                          history_dataset_association.id AS id),
+             hda_events
+          AS (INSERT INTO cleanup_event_hda_association
+                          (create_time, cleanup_event_id, hda_id)
+                   SELECT NOW() AT TIME ZONE 'utc', %(event_id)s, id
+                     FROM purged_hda_ids),
+             {purge_hda_dependencies_sql}
+      SELECT purged_user_ids.id AS purged_user_id,
+             purged_user_ids.id AS zero_disk_usage_user_id,
+             purged_history_ids.id AS purged_history_id,
+             purged_hda_ids.id AS purged_hda_id,
+             deleted_metadata_file_ids.id AS deleted_metadata_file_id,
+             deleted_metadata_file_ids.object_store_id AS object_store_id,
+             deleted_icda_ids.id AS deleted_icda_id,
+             deleted_icda_ids.hda_id AS deleted_icda_hda_id,
+             deleted_uga_ids.id AS deleted_uga_id,
+             deleted_ura_ids.id AS deleted_ura_id,
+             deleted_ua_ids.id AS deleted_ua_id
+        FROM purged_user_ids
+             LEFT OUTER JOIN purged_history_ids
+                             ON purged_user_ids.id = purged_history_ids.user_id
+             LEFT OUTER JOIN purged_hda_ids
+                             ON purged_history_ids.id = purged_hda_ids.history_id
+             LEFT OUTER JOIN deleted_metadata_file_ids
+                             ON deleted_metadata_file_ids.hda_id = purged_hda_ids.id
+             LEFT OUTER JOIN deleted_icda_ids
+                             ON deleted_icda_ids.hda_parent_id = purged_hda_ids.id
+             LEFT OUTER JOIN deleted_uga_ids
+                             ON purged_user_ids.id = deleted_uga_ids.user_id
+             LEFT OUTER JOIN deleted_ura_ids
+                             ON purged_user_ids.id = deleted_ura_ids.user_id
+             LEFT OUTER JOIN deleted_ua_ids
+                             ON purged_user_ids.id = deleted_ua_ids.user_id
+    ORDER BY purged_user_ids.id
+    """
+    causals = (
+        ('purged_user_id', 'purged_history_id'),
+        ('purged_history_id', 'purged_hda_id'),
+        ('purged_hda_id', 'deleted_metadata_file_id', 'object_store_id'),
+        ('purged_hda_id', 'deleted_icda_id', 'deleted_icda_hda_id'),
+        ('purged_user_id', 'deleted_uga_id'),
+        ('purged_user_id', 'deleted_ura_id'),
+        ('purged_user_id', 'deleted_ua_id'),
+    )
+
+    @classmethod
+    def name_c(cls):
+        return 'purge_deleted_users_gdpr'
+
+
 class PurgeDeletedHDAs(PurgesHDAs, RemovesMetadataFiles, RequiresDiskUsageRecalculation, Action):
     """
     - Mark purged all HistoryDatasetAssociations currently marked deleted that are older than the
@@ -1103,7 +1225,8 @@ def _dry_run_event(self):
 
     def _execute(self, sql, args):
         cur = self.conn.cursor()
-        log.debug("SQL is: %s", cur.mogrify(sql, args))
+        sql_str = cur.mogrify(sql, args).decode('utf-8')
+        log.debug(f"SQL is: {sql_str}")
         log.info("Executing SQL")
         cur.execute(sql, args)
         log.info('Database status: %s', cur.statusmessage)

test/unit/data/test_galaxy_mapping.py

@@ -1,5 +1,6 @@
 import collections
 import os
+import random
 import unittest
 import uuid
 from tempfile import NamedTemporaryFile
@@ -290,6 +291,42 @@ def test_collection_get_interface(self):
         for i in range(elements):
             assert c1[i] == dces[i]
 
+    def test_dataset_instance_order(self):
+        model = self.model
+        u = model.User(email="[email protected]", password="password")
+        h1 = model.History(name="History 1", user=u)
+        elements = []
+        list_pair = model.DatasetCollection(collection_type="list:paired")
+        for i in range(20):
+            pair = model.DatasetCollection(collection_type="pair")
+            forward = model.HistoryDatasetAssociation(extension="txt", history=h1, name=f"forward_{i}", create_dataset=True, sa_session=model.session)
+            reverse = model.HistoryDatasetAssociation(extension="bam", history=h1, name=f"reverse_{i}", create_dataset=True, sa_session=model.session)
+            dce1 = model.DatasetCollectionElement(collection=pair, element=forward, element_identifier=f"forward_{i}", element_index=1)
+            dce2 = model.DatasetCollectionElement(collection=pair, element=reverse, element_identifier=f"reverse_{i}", element_index=2)
+            to_persist = [(forward, reverse), (dce1, dce2)]
+            self.persist(pair)
+            for item in to_persist:
+                if i % 2:
+                    self.persist(item[0])
+                    self.persist(item[1])
+                else:
+                    self.persist(item[1])
+                    self.persist(item[0])
+            elements.append(model.DatasetCollectionElement(collection=list_pair, element=pair, element_index=i, element_identifier=str(i)))
+        self.persist(list_pair)
+        random.shuffle(elements)
+        for item in elements:
+            self.persist(item)
+        forward = []
+        reverse = []
+        for i, dataset_instance in enumerate(list_pair.dataset_instances):
+            if i % 2:
+                reverse.append(dataset_instance)
+            else:
+                forward.append(dataset_instance)
+        assert all(d.name == f"forward_{i}" for i, d in enumerate(forward))
+        assert all(d.name == f"reverse_{i}" for i, d in enumerate(reverse))
+
     def test_collections_in_histories(self):
         model = self.model