Merge pull request #10212 from jmchilton/quota_interface

Implement quota interface - with better logic isolation and unit tests.

client/src/layout/masthead.js

@@ -19,6 +19,7 @@ export class MastheadState {
         // add quota meter to masthead
         Galaxy.quotaMeter = this.quotaMeter = new QuotaMeter.UserQuotaMeter({
             model: Galaxy.user,
+            quotaUrl: Galaxy.config.quota_url,
         });
 
         // loop through beforeunload functions if the user attempts to unload the page

client/src/mvc/user/user-quotameter.js

@@ -20,6 +20,7 @@ var UserQuotaMeter = Backbone.View.extend(baseMVC.LoggableMixin).extend(
         options: {
             warnAtPercent: 85,
             errorAtPercent: 100,
+            quotaUrl: "https://galaxyproject.org/support/account-quotas/",
         },
 
         /** Set up, accept options, and bind events */
@@ -129,22 +130,17 @@ var UserQuotaMeter = Backbone.View.extend(baseMVC.LoggableMixin).extend(
         },
 
         _templateQuotaMeter: function (data) {
-            return [
-                '<div id="quota-meter" class="quota-meter progress">',
-                '<div class="progress-bar" style="width: ',
-                data.quota_percent,
-                '%"></div>',
-                '<div class="quota-meter-text" data-placement="left"',
-                data.nice_total_disk_usage ? ` title="Using ${data.nice_total_disk_usage}. Click for details.">` : ">",
-                '<a href="https://galaxyproject.org/support/account-quotas/" target="_blank">',
-                _l("Using"),
-                " ",
-                data.quota_percent,
-                "%",
-                "</a>",
-                "</div>",
-                "</div>",
-            ].join("");
+            const title = data.nice_total_disk_usage
+                ? `title="Using ${data.nice_total_disk_usage}. Click for details."`
+                : "";
+            const using = `${_l("Using")} ${data.quota_percent}%`;
+            const quotaUrl = this.options.quotaUrl;
+            return `<div id="quota-meter" class="quota-meter progress">
+    <div class="progress-bar" style="width: ${data.quota_percent}%"></div>
+    <div class="quota-meter-text" data-placement="left" ${title}>
+        <a href="${quotaUrl}" target="_blank">${using}</a>
+    </div>
+</div>`;
         },
 
         _templateUsage: function (data) {

lib/galaxy/app.py

@@ -6,7 +6,6 @@
 import galaxy.model
 import galaxy.model.security
 import galaxy.queues
-import galaxy.quota
 import galaxy.security
 from galaxy import config, job_metrics, jobs
 from galaxy.config_watchers import ConfigWatchers
@@ -27,6 +26,7 @@
     GalaxyQueueWorker,
     send_local_control_task,
 )
+from galaxy.quota import get_quota_agent
 from galaxy.tool_shed.galaxy_install.installed_repository_manager import InstalledRepositoryManager
 from galaxy.tool_shed.galaxy_install.update_repository_manager import UpdateRepositoryManager
 from galaxy.tool_util.deps.views import DependencyResolversView
@@ -175,10 +175,7 @@ def __init__(self, **kwargs):
             model=self.security_agent.model,
             permitted_actions=self.security_agent.permitted_actions)
         # Load quota management.
-        if self.config.enable_quotas:
-            self.quota_agent = galaxy.quota.QuotaAgent(self.model)
-        else:
-            self.quota_agent = galaxy.quota.NoQuotaAgent(self.model)
+        self.quota_agent = get_quota_agent(self.config, self.model)
         # Heartbeat for thread profiling
         self.heartbeat = None
         from galaxy import auth

lib/galaxy/jobs/handler.py

@@ -578,15 +578,8 @@ def __verify_job_ready(self, job, job_wrapper):
 
         if state == JOB_READY:
             state = self.__check_user_jobs(job, job_wrapper)
-        if state == JOB_READY and self.app.config.enable_quotas:
-            quota = self.app.quota_agent.get_quota(job.user)
-            if quota is not None:
-                try:
-                    usage = self.app.quota_agent.get_usage(user=job.user, history=job.history)
-                    if usage > quota:
-                        return JOB_USER_OVER_QUOTA, job_destination
-                except AssertionError:
-                    pass  # No history, should not happen with an anon user
+        if state == JOB_READY and self.app.quota_agent.is_over_quota(self.app, job, job_destination):
+            return JOB_USER_OVER_QUOTA, job_destination
         # Check total walltime limits
         if (state == JOB_READY and
                 "delta" in self.app.job_config.limits.total_walltime):

lib/galaxy/managers/configuration.py

@@ -54,6 +54,7 @@ def _use_config(config, key, **context):
             'screencasts_url'                   : _use_config,
             'citation_url'                      : _use_config,
             'support_url'                       : _use_config,
+            'quota_url'                         : _use_config,
             'helpsite_url'                      : _use_config,
             'lims_doc_url'                      : _defaults_to("https://usegalaxy.org/u/rkchak/p/sts"),
             'default_locale'                    : _use_config,

lib/galaxy/managers/users.py

@@ -349,7 +349,7 @@ def default_permissions(self, user):
 
     def quota(self, user, total=False):
         if total:
-            return self.app.quota_agent.get_quota(user, nice_size=True)
+            return self.app.quota_agent.get_quota_nice_size(user)
         return self.app.quota_agent.get_percent(user=user)
 
     def tags_used(self, user, tag_models=None):

lib/galaxy/quota/__init__.py

@@ -1,26 +1,42 @@
-"""
-Galaxy Quotas
-"""
+"""Galaxy Quotas"""
+import abc
 import logging
 
 import galaxy.util
 
 log = logging.getLogger(__name__)
 
 
-class NoQuotaAgent:
-    """Base quota agent, always returns no quota"""
+class QuotaAgent(metaclass=abc.ABCMeta):
+    """Abstraction around querying Galaxy for quota available and used.
 
-    def __init__(self, model):
-        self.model = model
-        self.sa_session = model.context
+    Certain parts of the app that deal directly with modifying the quota assume more than
+    this interface - they assume the availability of the methods on DatabaseQuotaAgent that
+    implements this interface. But for read-only quota operations - such as checking available
+    quota or reporting it to users - methods defined on this interface should be sufficient
+    and the NoQuotaAgent should be a valid choice.
 
-    def get_quota(self, user, nice_size=False):
-        return None
+    Sticking to well annotated methods on this interface should make it clean and
+    possible to implement other backends for quota setting in the future such as managing
+    the quota in other apps (LDAP maybe?) or via configuration files.
+    """
 
-    @property
-    def default_quota(self):
-        return None
+    @abc.abstractmethod
+    def get_quota(self, user):
+        """Return quota in bytes or None if no quota is set."""
+
+    def get_quota_nice_size(self, user):
+        """Return quota as a human-readable string or 'unlimited' if no quota is set."""
+        quota_bytes = self.get_quota(user)
+        if quota_bytes is not None:
+            quota_str = galaxy.util.nice_size(quota_bytes)
+        else:
+            quota_str = 'unlimited'
+        return quota_str
+
+    @abc.abstractmethod
+    def get_percent(self, trans=None, user=False, history=False, usage=False, quota=False):
+        """Return the percentage of any storage quota applicable to the user/transaction."""
 
     def get_usage(self, trans=None, user=False, history=False):
         if trans:
@@ -34,17 +50,43 @@ def get_usage(self, trans=None, user=False, history=False):
             usage = user.total_disk_usage
         return usage
 
+    def is_over_quota(self, app, job, job_destination):
+        """Return True if the user or history is over quota for specified job.
+
+        job_destination unused currently but an important future application will
+        be admins and/or users dynamically specifying which object stores to use
+        and that will likely come in through the job destination.
+        """
+
+
+class NoQuotaAgent(QuotaAgent):
+    """Base quota agent, always returns no quota"""
+
+    def __init__(self):
+        pass
+
+    def get_quota(self, user):
+        return None
+
+    @property
+    def default_quota(self):
+        return None
+
     def get_percent(self, trans=None, user=False, history=False, usage=False, quota=False):
         return None
 
-    def get_user_quotas(self, user):
-        return []
+    def is_over_quota(self, app, job, job_destination):
+        return False
 
 
-class QuotaAgent(NoQuotaAgent):
+class DatabaseQuotaAgent(QuotaAgent):
     """Class that handles galaxy quotas"""
 
-    def get_quota(self, user, nice_size=False):
+    def __init__(self, model):
+        self.model = model
+        self.sa_session = model.context
+
+    def get_quota(self, user):
         """
         Calculated like so:
 
@@ -92,11 +134,6 @@ def get_quota(self, user, nice_size=False):
             rval = max + adjustment
             if rval <= 0:
                 rval = 0
-        if nice_size:
-            if rval is not None:
-                rval = galaxy.util.nice_size(rval)
-            else:
-                rval = 'unlimited'
         return rval
 
     @property
@@ -179,21 +216,24 @@ def set_entity_quota_associations(self, quotas=None, users=None, groups=None, de
                 self.sa_session.add(gqa)
             self.sa_session.flush()
 
-    def get_user_quotas(self, user):
-        rval = []
-        if not user:
-            dqa = self.sa_session.query(self.model.DefaultQuotaAssociation) \
-                .filter(self.model.DefaultQuotaAssociation.table.c.type == self.model.DefaultQuotaAssociation.types.UNREGISTERED).first()
-            if dqa:
-                rval.append(dqa.quota)
-        else:
-            dqa = self.sa_session.query(self.model.DefaultQuotaAssociation) \
-                .filter(self.model.DefaultQuotaAssociation.table.c.type == self.model.DefaultQuotaAssociation.types.REGISTERED).first()
-            if dqa:
-                rval.append(dqa.quota)
-            for uqa in user.quotas:
-                rval.append(uqa.quota)
-            for group in [uga.group for uga in user.groups]:
-                for gqa in group.quotas:
-                    rval.append(gqa.quota)
-        return rval
+    def is_over_quota(self, app, job, job_destination):
+        quota = self.get_quota(job.user)
+        if quota is not None:
+            try:
+                usage = self.get_usage(user=job.user, history=job.history)
+                if usage > quota:
+                    return True
+            except AssertionError:
+                pass  # No history, should not happen with an anon user
+        return False
+
+
+def get_quota_agent(config, model):
+    if config.enable_quotas:
+        quota_agent = galaxy.quota.DatabaseQuotaAgent(model)
+    else:
+        quota_agent = galaxy.quota.NoQuotaAgent()
+    return quota_agent
+
+
+__all__ = ('get_quota_agent', 'NoQuotaAgent')

lib/galaxy/webapps/galaxy/config_schema.yml

@@ -1276,6 +1276,13 @@ mapping:
         desc: |
           The URL linked by the "Wiki" link in the "Help" menu.
 
+      quota_url:
+        type: str
+        default: https://galaxyproject.org/support/account-quotas/
+        required: false
+        desc: |
+          The URL linked for quota information in the UI.
+
       support_url:
         type: str
         default: https://galaxyproject.org/support/

lib/galaxy_test/base/populators.py

@@ -549,6 +549,16 @@ def create_role(self, user_ids, description=None):
         assert role_response.status_code == 200
         return role_response.json()[0]
 
+    def create_quota(self, quota_payload):
+        quota_response = self.galaxy_interactor.post("quotas", data=quota_payload, admin=True)
+        quota_response.raise_for_status()
+        return quota_response.json()
+
+    def get_quotas(self):
+        quota_response = self.galaxy_interactor.get("quotas", admin=True)
+        quota_response.raise_for_status()
+        return quota_response.json()
+
     def make_private(self, history_id, dataset_id):
         role_id = self.user_private_role_id()
         # Give manage permission to the user.

lib/tool_shed/webapp/app.py

@@ -3,13 +3,13 @@
 import time
 
 import galaxy.datatypes.registry
-import galaxy.quota
 import galaxy.tools.data
 import tool_shed.repository_registry
 import tool_shed.repository_types.registry
 import tool_shed.webapp.model
 from galaxy.config import configure_logging
 from galaxy.model.tags import CommunityTagHandler
+from galaxy.quota import NoQuotaAgent
 from galaxy.security import idencoding
 from galaxy.util.dbkeys import GenomeBuilds
 from galaxy.web_stack import application_stack_instance
@@ -69,7 +69,7 @@ def __init__(self, **kwd):
         # Initialize the Tool Shed security agent.
         self.security_agent = self.model.security_agent
         # The Tool Shed makes no use of a quota, but this attribute is still required.
-        self.quota_agent = galaxy.quota.NoQuotaAgent(self.model)
+        self.quota_agent = NoQuotaAgent()
         # Initialize the baseline Tool Shed statistics component.
         self.shed_counter = self.model.shed_counter
         # Let the Tool Shed's HgwebConfigManager know where the hgweb.config file is located.

test/integration/test_quota.py

@@ -0,0 +1,32 @@
+from galaxy_test.base.populators import (
+    DatasetPopulator,
+)
+from galaxy_test.driver import integration_util
+
+
+class QuotaIntegrationTestCase(integration_util.IntegrationTestCase):
+    require_admin_user = True
+
+    @classmethod
+    def handle_galaxy_config_kwds(cls, config):
+        config["enable_quotas"] = True
+
+    def setUp(self):
+        super().setUp()
+        self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
+
+    def test_quota_crud(self):
+        quotas = self.dataset_populator.get_quotas()
+        assert len(quotas) == 0
+
+        payload = {
+            'name': 'defaultquota1',
+            'description': 'first default quota',
+            'amount': '100MB',
+            'operation': '=',
+            'default': 'registered',
+        }
+        self.dataset_populator.create_quota(payload)
+
+        quotas = self.dataset_populator.get_quotas()
+        assert len(quotas) == 1