diff --git a/bin/create_muk_scratch_directory_tree.py b/bin/create_muk_scratch_directory_tree.py deleted file mode 100644 index d8d46218..00000000 --- a/bin/create_muk_scratch_directory_tree.py +++ /dev/null @@ -1,137 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2012-2017 Ghent University -# -# This file is part of vsc-administration, -# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), -# with support of Ghent University (http://ugent.be/hpc), -# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), -# the Flemish Research Foundation (FWO) (http://www.fwo.be/en) -# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). -# -# https://github.com/hpcugent/vsc-administration -# -# All rights reserved. -# -""" -Sets up the baseline directory structure we need to have on muk. - -- a user fileset -- an apps fileset -- creates symlinks to these if they do not yet exist (on the node where the script is being run - - /user -> /gpfs/scratch/user - - /apps -> /gpfs/scratch/apps - -@author: Andy Georges -""" - -import os - -from vsc.filesystem.gpfs import GpfsOperations -from vsc.ldap.filters import CnFilter -from vsc.ldap.utils import LdapQuery -from vsc.ldap.configuration import LumaConfiguration -from vsc.administration.user import MukAccountpageUser -from vsc.config.base import Muk -from vsc.utils import fancylogger - -log = fancylogger.getLogger('create_directory_trees_muk') - -PILOT_PROJECTS = { - 'a': 'project_apilot', - 'b': 'project_bpilot', - 'g': 'project_gpilot', - 'l': 'project_lpilot', -} - - -def main(): - """Main.""" - - ldap_query = LdapQuery(LumaConfiguration()) # initialise - muk = Muk() - - gpfs = GpfsOperations() - gpfs.list_filesets() - scratch = gpfs.get_filesystem_info(muk.scratch_name) - - # Create the base user fileset that will be used to store the directory - # hierarchy to mimic the login/home directories of users - user_fileset_path = os.path.join(scratch['defaultMountPoint'], 'user') - if not 'user' in [f['filesetName'] for f in gpfs.gpfslocalfilesets[muk.scratch_name].values()]: - gpfs.make_fileset(user_fileset_path, 'user') - gpfs.chmod(0o755, user_fileset_path) - log.info("Fileset user created and linked at %s" % (user_fileset_path)) - - # Create the applications fileset that will be used for storing apps and tools - # that can be used by the users on muk - apps_fileset_path = os.path.join(scratch['defaultMountPoint'], 'apps') - if not 'apps' in [f['filesetName'] for f in gpfs.gpfslocalfilesets[muk.scratch_name].values()]: - gpfs.make_fileset(apps_fileset_path, 'apps') - gpfs.chmod(0o755, apps_fileset_path) - log.info("Fileset apps created and linked at %s" % (apps_fileset_path)) - - # Create the projects fileset that will be used to store the directory - # hierarchy for all project spaces on muk scratch - projects_fileset_path = os.path.join(scratch['defaultMountPoint'], 'projects') - if not 'projects' in [f['filesetName'] for f in gpfs.gpfslocalfilesets[muk.scratch_name].values()]: - gpfs.make_fileset(projects_fileset_path, 'projects') - gpfs.chmod(0o755, projects_fileset_path) - log.info("Fileset projects created and linked at %s" % (projects_fileset_path)) - - # If users are to log in, there should be a symlink to the GPFS directory hierarchy - if not os.path.lexists('/user'): - os.symlink(user_fileset_path, '/user') - log.info("Linking /user to %s" % (user_fileset_path)) - - # If the apps are to be reachable in a similar vein as on the Tier-2, we need a symlink - # from the FS root to the real fileset path - if not os.path.lexists('/apps'): - os.symlink(apps_fileset_path, '/apps') - log.info("Linking /apps to %s" % (apps_fileset_path)) - - # In the pilot phase, we have 4 project filesets that need to be owned by the pilot groups - # moderator and be group rw for said pilot group - pilot_projects = PILOT_PROJECTS - - for institute in pilot_projects.keys(): - group_name = "%st1_mukusers" % institute - try: - group = ldap_query.group_filter_search(CnFilter(group_name))[0] - except Exception: - log.error("No LDAP group with the name %s found" % (group_name)) - continue - - owner = ldap_query.user_filter_search(CnFilter(group['moderator'][0]))[0] - - project_fileset_name = pilot_projects[institute] - project_fileset_path = os.path.join(scratch['defaultMountPoint'], 'projects', project_fileset_name) - - if not project_fileset_name in [f['filesetName'] for f in gpfs.gpfslocalfilesets[muk.scratch_name].values()]: - try: - gpfs.make_fileset(project_fileset_path, project_fileset_name) - log.info("Created new fileset %s with link path %s" % (project_fileset_name, project_fileset_path)) - except Exception: - log.exception("Failed to create a new fileset with the name %s and link path %s" % - (project_fileset_name, project_fileset_path)) - - gpfs.chmod(0o755, project_fileset_path) - os.chown(project_fileset_path, int(owner['uidNumber']), int(group['gidNumber'])) - - project_quota = 70 * 1024 * 1024 * 1024 * 1024 - gpfs.set_fileset_quota(project_quota, project_fileset_path, project_fileset_name) - - -def add_example_users(): - """Usage example on how to add a user. - - This creates paths and filesets for an UGent user with regular home on the gengar storage, NFS mounted - """ - u = MukAccountpageUser('vsc40075') # ageorges - u.create_scratch_fileset() - u.populate_scratch_fallback() # we should always do this, so we can shift the symlinks around at leisure. - u.create_home_dir() # this creates the symlink from the directory hierarchy in the scratch to the actual home - - -if __name__ == '__main__': - main() diff --git a/bin/sync_muk_users.py b/bin/sync_muk_users.py deleted file mode 100644 index b634d105..00000000 --- a/bin/sync_muk_users.py +++ /dev/null @@ -1,429 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2012-2017 Ghent University -# -# This file is part of vsc-administration, -# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), -# with support of Ghent University (http://ugent.be/hpc), -# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), -# the Flemish Research Foundation (FWO) (http://www.fwo.be/en) -# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). -# -# https://github.com/hpcugent/vsc-administration -# -# All rights reserved. -# -""" -This script checks the users entries in the LDAP that have changed since a given timestamp -and that are in the muk autogroup. - -For these, the home and other shizzle should be set up. - -@author Andy Georges -""" - -import logging -import os -import sys -import time - -from urllib2 import HTTPError - -from vsc.accountpage.client import AccountpageClient -from vsc.administration.user import MukAccountpageUser -from vsc.administration.tools import cleanup_purgees -from vsc.administration.tools import TIER1_GRACE_GROUP_SUFFIX, TIER1_HELPDESK_ADDRESS, UGENT_SMTP_ADDRESS -from vsc.config.base import Muk -from vsc.utils import fancylogger -from vsc.utils.cache import FileCache -from vsc.utils.mail import VscMail, VscMailError -from vsc.utils.nagios import NAGIOS_EXIT_CRITICAL -from vsc.utils.script_tools import ExtendedSimpleOption - -NAGIOS_HEADER = 'sync_muk_users' -NAGIOS_CHECK_INTERVAL_THRESHOLD = 15 * 60 # 15 minutes - -SYNC_TIMESTAMP_FILENAME = "/var/run/%s.timestamp" % (NAGIOS_HEADER) -SYNC_MUK_USERS_LOCKFILE = "/gpfs/scratch/user/%s.lock" % (NAGIOS_HEADER) - -PURGE_NOTIFICATION_TIMES = (7 * 86400, 12 * 86400) -PURGE_DEADLINE_TIME = 14 * 86400 - -logger = fancylogger.getLogger(__name__) -fancylogger.logToScreen(True) -fancylogger.setLogLevelInfo() - - -GRACE_MESSAGE = """ -Dear %(gecos)s, - -Your allocated compute time on the VSC Tier-1 cluster at Ghent has expired. -You are now in a grace state for the next %(grace_time)s. This means you can -no longer submit new jobs to the scheduler. Jobs running at this moment will -not be killed and should likely finish. - -Please make sure you copy back all required results from the dedicated -$VSC_SCRATCH storage on the Tier-1 to your home institution's long term -storage, since you will no longer be able to log in to this machine once -the grace period expires. - -Should you have any questions, please contact us at %(tier1_helpdesk)s or reply to -this email which will open a ticket in our helpdesk system for you. - -Kind regards, --- The UGent HPC team -""" - -FINAL_MESSAGE = """ -Dear %(gecos)s, - -The grace period for your compute time on the VSC Tier-1 cluster at Ghent -has expired. From this point on, you will not be able to log in to the -machine anymore, nor will you be able to reach its dedicated $VSC_SCRATCH -storage. - -Should you have any questions, please contact us at %(tier1_helpdesk)s or reply to -this email which will open a ticket in our helpdesk system for you. - -Kind regards, --- The UGent HPC team -""" - - -def process_institute(options, institute, institute_users, client): - """ - Sync the users from the given institute to the system - """ - muk = Muk() # Singleton class, so no biggie - - try: - nfs_location = muk.nfs_link_pathnames[institute]['home'] - logger.info("Checking link to NFS mount at %s" % (nfs_location)) - os.stat(nfs_location) - try: - error_users = process(options, institute_users, client) - except Exception: - logger.exception("Something went wrong processing users from %s" % (institute)) - error_users = institute_users - except Exception: - logger.exception("Cannot process users from institute %s, cannot stat link to NFS mount" % (institute)) - error_users = institute_users - - fail_usercount = len(error_users) - ok_usercount = len(institute_users) - fail_usercount - - return { - 'ok': ok_usercount, - 'fail': fail_usercount, - } - - -def process(options, users, client): - """ - Actually do the tasks for a changed or new user: - - - create the user's fileset - - set the quota - - create the home directory as a link to the user's fileset - """ - - error_users = [] - for user_id in sorted(users): - user = MukAccountpageUser(user_id, rest_client=client) - user.dry_run = options.dry_run - try: - user.create_scratch_fileset() - user.populate_scratch_fallback() - user.create_home_dir() - except Exception: - logger.exception("Cannot process user %s" % (user_id)) - error_users.append(user_id) - - return error_users - - -def force_nfs_mounts(muk): - """ - Make sure that the NFS mounts are there - """ - - nfs_mounts = [] - for institute in muk.institutes: - try: - os.stat(muk.nfs_link_pathnames[institute]['home']) - nfs_mounts.append(institute) - except Exception: - logger.exception("Cannot stat %s, not adding institute" % muk.nfs_link_pathnames[institute]['home']) - - return nfs_mounts - - -def add_users_to_purgees(previous_users, current_users, purgees, now, client, dry_run): - """ - Add the users that are out to the purgees - """ - purgees_first_notify = 0 - for user_id in previous_users: - user = MukAccountpageUser(user_id, rest_client=client) - user.dry_run = dry_run - if user_id not in current_users and user_id not in purgees: - if not user.dry_run: - group_name = "%st1_mukgraceusers" % user.person.institute['site'][0] - try: - client.group[group_name].member[user_id].post() - except HTTPError as err: - logging.error( - "Return code %d: could not add %s to group %s [%s]. Not notifying user or adding to purgees.", - err.code, user_id, group_name, err) - continue - else: - logging.info("Account %s added to group %s", user_id, group_name) - notify_user_of_purge(user, now, now) - purgees[user_id] = (now, None, None) # in a dry run we will not store these in the cache - purgees_first_notify += 1 - logger.info("Added %s to the list of purgees with timestamp %s" % (user_id, (now, None, None))) - - else: - logger.info("User %s in both previous users and current users lists, not eligible for purge." % (user_id,)) - - return purgees_first_notify - - -def purge_obsolete_symlinks(path, current_users, client, dry_run): - """ - The symbolic links to home directories must vanish for people who no longer have access. - - For this we use a cache with the following items. - - previous list of tier-1 members - - to-be-purged dict of member, timestamp pairs - - @type path: string - @type current_users: list of user login names - - @param path: path to the cache of purged or to be purged users - @param current_users: VSC members who are entitled to compute on the Tier-1 at this point in time - """ - - now = time.time() - cache = FileCache(path) - - purgees_undone = 0 - purgees_first_notify = 0 - purgees_second_notify = 0 - purgees_final_notify = 0 - purgees_begone = 0 - - previous_users = cache.load('previous_users') - if previous_users: - (_, previous_users) = previous_users - else: - logger.warning("Purge cache has no previous_users") - previous_users = [] - - purgees = cache.load('purgees') - if purgees: - (_, purgees) = purgees - else: - logger.warning("Purge cache has no purgees") - purgees = dict() - - logger.info("Starting purge at time %s" % (now,)) - logger.debug("Previous users: %s", (previous_users,)) - logger.debug("Purgees: %s", (purgees,)) - - # if a user is added again before his grace ran out, remove him from the purgee list and from the grace group - purgees_undone = cleanup_purgees(current_users, purgees, client, dry_run) - - # warn those still on the purge list if needed - for (user_id, (first_warning, second_warning, final_warning)) in purgees.items(): - logger.debug("Checking if we should warn %s at %d, time since purge: %d", user_id, now, now - first_warning) - - user = MukAccountpageUser(user_id, rest_client=client) - user.dry_run = dry_run - if now - first_warning > PURGE_DEADLINE_TIME: - notify_user_of_purge(user, first_warning, now) - purge_user(user, client) - del purgees[user_id] - purgees_begone += 1 - logger.info("Removed %s from the list of purgees - time's up " % (user_id, )) - elif not second_warning and now - first_warning > PURGE_NOTIFICATION_TIMES[0]: - notify_user_of_purge(user, first_warning, now) - purgees[user_id] = (first_warning, now, None) - purgees_second_notify += 1 - logger.info("Updated %s in the list of purgees with timestamps %s" % (user_id, (first_warning, now, None))) - elif not final_warning and now - first_warning > PURGE_NOTIFICATION_TIMES[1]: - notify_user_of_purge(user, first_warning, now) - purgees[user_id] = (first_warning, second_warning, now) - purgees_final_notify += 1 - logger.info("Updated %s in the list of purgees with timestamps %s" % - (user_id, (first_warning, second_warning, now))) - else: - logger.info("Time difference does not warrant sending a new mail already.") - - # add those that went to the other side and warn them - purgees_first_notify = add_users_to_purgees(previous_users, current_users, purgees, now, client, dry_run) - - if not dry_run: - cache.update('previous_users', current_users, 0) - cache.update('purgees', purgees, 0) - logger.info("Purge cache updated") - else: - logger.info("Dry run: not updating the purgees cache") - - cache.close() - - return { - 'purgees_undone': purgees_undone, - 'purgees_first_notify': purgees_first_notify, - 'purgees_second_notify': purgees_second_notify, - 'purgees_final_notify': purgees_final_notify, - 'purgees_begone': purgees_begone, - } - - -def notify_user_of_purge(user, grace_time, current_time): - """ - Send out a notification mail to the user letting him know he will be purged in n days or k hours. - - @type user: MukAccountpageUser - """ - time_left = grace_time + PURGE_DEADLINE_TIME - current_time - - logger.debug("Time time_left for %s: %d seconds", user, time_left) - - if time_left < 0: - time_left = 0 - left_unit = None - if time_left <= 86400: - time_left /= 3600 - left_unit = "hours" - else: - time_left /= 86400 - left_unit = "days" - - logger.info("Sending notification mail to %s - time time_left before purge %d %s" % (user, time_left, left_unit)) - if time_left: - notify_purge(user, time_left, left_unit) - else: - notify_purge(user, None, None) - - -def notify_purge(user, grace=0, grace_unit=""): - """Send out the actual notification""" - mail = VscMail(mail_host=UGENT_SMTP_ADDRESS) - - if grace: - message = GRACE_MESSAGE % ({'gecos': user.person.gecos, - 'grace_time': "%d %s" % (grace, grace_unit), - 'tier1_helpdesk': TIER1_HELPDESK_ADDRESS, - }) - mail_subject = "%s compute on the VSC Tier-1 entering grace period" % user.account.vsc_id - - else: - message = FINAL_MESSAGE % ({'gecos': user.person.gecos, - 'tier1_helpdesk': TIER1_HELPDESK_ADDRESS, - }) - mail_subject = "%(user_id)s compute time on the VSC Tier-1 expired" % ({'user_id': user.account.vsc_id}) - - if user.dry_run: - logger.info("Dry-run, would send the following message to %s: %s" % (user.account.vsc_id, message,)) - else: - try: - mail.sendTextMail(mail_to=user.account.email, - mail_from=TIER1_HELPDESK_ADDRESS, - reply_to=TIER1_HELPDESK_ADDRESS, - mail_subject=mail_subject, - message=message) - logger.info("notification: recipient %s [%s] sent expiry mail with subject %s" % - (user.account.vsc_id, user.person.gecos, mail_subject)) - except VscMailError as err: - logger.error("Sending mail to %s (via %s) failed: %s", err.mail_to, err.mail_host, err.err) - - -def purge_user(user, client): - """ - Really purge the user by removing the symlink to his home dir. - """ - if not user.dry_run: - logger.info("Purging %s" % (user.account.vsc_id,)) - # remove the user from the grace users - group_name = user.get_institute_prefix() + TIER1_GRACE_GROUP_SUFFIX - try: - client.group[group_name].member[user.account.vsc_id].delete() - except HTTPError as err: - logging.error("Return code %d: could not remove %s from group %s [%s]", - err.code, user.account.vsc_id, group_name, err) - else: - logging.info("Account %s removed from group %s", user.account.vsc_id, group_name) - - user.cleanup_home_dir() - else: - logging.info("Dry-run: not removing user %s from grace group" % (user.account.vsc_id,)) - logging.info("Dry-run: not cleaning up home dir symlink for user %s" % (user.account.vsc_id)) - - -def main(): - """ - Main script. - - loads the previous timestamp - - build the filter - - fetches the users - - process the users - - write the new timestamp if everything went OK - - write the nagios check file - """ - - options = { - 'nagios-check-interval-threshold': NAGIOS_CHECK_INTERVAL_THRESHOLD, - 'locking-filename': SYNC_MUK_USERS_LOCKFILE, - 'purge-cache': ('Location of the cache with users that should be purged', None, 'store', None), - 'access_token': ('OAuth2 token identifying the user with the accountpage', None, 'store', None), - } - - opts = ExtendedSimpleOption(options) - stats = {} - - try: - muk = Muk() - nfs_mounts = force_nfs_mounts(muk) - logger.info("Forced NFS mounts") - - client = AccountpageClient(token=opts.options.access_token) - - muk_users_set = client.autogroup[muk.muk_user_group].get()[1]['members'] - logger.debug("Found the following Muk users: {users}".format(users=muk_users_set)) - - for institute in nfs_mounts: - - (status, institute_users) = client.account.institute[institute].get() - if status == 200: - muk_institute_users = set([u['vsc_id'] for u in institute_users]).intersection(muk_users_set) - users_ok = process_institute(opts.options, institute, muk_institute_users, client) - else: - # not sure what to do here. - continue - - total_institute_users = len(muk_institute_users) - stats["%s_users_sync" % (institute,)] = users_ok.get('ok', 0) - # 20% of all users want to get on - stats["%s_users_sync_warning" % (institute,)] = int(total_institute_users / 5) - # 30% of all users want to get on - stats["%s_users_sync_critical" % (institute,)] = int(total_institute_users / 2) - stats["%s_users_sync_fail" % (institute,)] = users_ok.get('fail', 0) - stats["%s_users_sync_fail_warning" % (institute,)] = users_ok.get('fail', 0) - stats["%s_users_sync_fail_warning" % (institute,)] = 1 - stats["%s_users_sync_fail_critical" % (institute,)] = 3 - - purgees_stats = purge_obsolete_symlinks(opts.options.purge_cache, muk_users_set, client, opts.options.dry_run) - stats.update(purgees_stats) - - except Exception as err: - logger.exception("critical exception caught: %s" % (err)) - opts.critical("Script failed in a horrible way") - sys.exit(NAGIOS_EXIT_CRITICAL) - - opts.epilogue("Muk users synchronisation completed", stats) - - -if __name__ == '__main__': - main() diff --git a/lib/vsc/administration/tools.py b/lib/vsc/administration/tools.py index 1ec85aa6..77c70e53 100644 --- a/lib/vsc/administration/tools.py +++ b/lib/vsc/administration/tools.py @@ -24,8 +24,6 @@ import os import stat -from urllib2 import HTTPError - from vsc.utils import fancylogger from vsc.utils.mail import VscMail @@ -78,61 +76,3 @@ def create_stat_directory(path, permissions, uid, gid, posix, override_permissio logging.debug("Path %s already exists with correct ownership" % (path,)) return created - - -def notify_reinstatement(user): - """ - Send out a mail notifying the user who was removed from grace and back to regular mode on muk. - - @type user: MukAccountpageUser - """ - mail = VscMail(mail_host=UGENT_SMTP_ADDRESS) - - message = REINSTATEMENT_MESSAGE % ({'gecos': user.person.gecos, - 'tier1_helpdesk': TIER1_HELPDESK_ADDRESS, - }) - mail_subject = "%(user_id)s VSC Tier-1 access reinstated" % ({'user_id': user.account.vsc_id}) - - if user.dry_run: - logger.info("Dry-run, would send the following message to %s: %s" % (user, message,)) - else: - mail.sendTextMail(mail_to=user.account.email, - mail_from=TIER1_HELPDESK_ADDRESS, - reply_to=TIER1_HELPDESK_ADDRESS, - mail_subject=mail_subject, - message=message) - logger.info("notification: recipient %s [%s] sent expiry mail with subject %s" % - (user.account.vsc_id, user.person.gecos, mail_subject)) - - -def cleanup_purgees(current_users, purgees, client, dry_run): - """ - Remove users from purgees if they are in the current users list. - """ - from vsc.administration.user import MukAccountpageUser - purgees_undone = 0 - for user_id in current_users: - logger.debug("Checking if %s is in purgees", (user_id,)) - if user_id in purgees: - del purgees[user_id] - purgees_undone += 1 - logger.info("Removed %s from the list of purgees: found in list of current users" % (user_id,)) - user = MukAccountpageUser(user_id, rest_client=client) - user.dry_run = dry_run - notify_reinstatement(user) - - group_name = "%st1_mukgraceusers" % user.person.institute['site'][0] # just the first letter - if not user.dry_run: - try: - client.group[group_name].member[user.account.vsc_id].delete() - except HTTPError, err: - logging.error("Return code %d: could not remove %s from group %s [%s].", - err.code, user.account.vsc_id, group_name, err) - continue - else: - logging.info("Account %s removed to group %s", user.account.vsc_id, group_name) - else: - logging.info("Dry-run: not removing user %s from grace users group %s" % - (user.account.vsc_id, group_name)) - - return purgees_undone diff --git a/lib/vsc/administration/user.py b/lib/vsc/administration/user.py index bc2ccff5..4a16fc1e 100644 --- a/lib/vsc/administration/user.py +++ b/lib/vsc/administration/user.py @@ -20,7 +20,6 @@ @author: Andy Georges (Ghent University) """ -import errno import logging import os @@ -31,9 +30,8 @@ from vsc.accountpage.wrappers import mkVscAccount, mkUserGroup from vsc.accountpage.wrappers import mkGroup, mkVscUserSizeQuota from vsc.administration.tools import create_stat_directory -from vsc.config.base import VSC, Muk, VscStorage, VSC_DATA, VSC_HOME, GENT_PRODUCTION_SCRATCH +from vsc.config.base import VSC, VscStorage, VSC_DATA, VSC_HOME, GENT_PRODUCTION_SCRATCH from vsc.config.base import NEW, MODIFIED, MODIFY, ACTIVE -from vsc.filesystem.ext import ExtOperations from vsc.filesystem.gpfs import GpfsOperations from vsc.filesystem.posix import PosixOperations @@ -172,7 +170,9 @@ def user_proposition(quota, storage_type): return quota.fileset == fileset_name and quota.storage['storage_type'] == storage_type self._quota_cache['home'] = [q.hard for q in institute_quota if user_proposition(q, 'home')][0] - self._quota_cache['data'] = [q.hard for q in institute_quota if user_proposition(q, 'data')][0] + self._quota_cache['data'] = [q.hard for q in institute_quota + if user_proposition(q, 'data') + and not q.storage['name'].endswith('SHARED')][0] self._quota_cache['scratch'] = filter(lambda q: user_proposition(q, 'scratch'), institute_quota) fileset_name = 'gvo' @@ -400,180 +400,13 @@ def __setattr__(self, name, value): super(VscTier2AccountpageUser, self).__setattr__(name, value) -class MukAccountpageUser(VscAccountPageUser): - """A VSC user who is allowed to execute on the Tier 1 machine(s). - - This class provides functionality for administrating users on the - Tier 1 machine(s). - - - Provide a fileset for the user on scratch ($VSC_SCRATCH) - - Set up quota (scratch) - - Symlink the user's home ($VSC_HOME) to the real home - - AFM cached mount (GPFS) of the NFS path - - NFS mount of the home institute's directory for the user - - Local scratch location - This is more involved than it seems since Ghent has a different - path compared to the other institutes. - Also, /scratch needs to remain the real scratch. - - All changes should be an idempotent operation, i.e., f . f = f. - - All changes should be made based on the timestamp of the LDAP entry, - i.e., only if the modification time is more recent, we update the - deployed settings. - """ - - def __init__(self, user_id, storage=None, pickle_storage='VSC_SCRATCH_MUK', rest_client=None): - """Initialisation. - @type vsc_user_id: string representing the user's VSC ID (vsc[0-9]{5}) - """ - super(MukAccountpageUser, self).__init__(user_id, rest_client) - - if not storage: - self.storage = VscStorage() - else: - self.storage = storage - - self.gpfs = GpfsOperations() # Only used when needed - self.posix = PosixOperations() - self.ext = ExtOperations() - - self.pickle_storage = pickle_storage - - self.muk = Muk() - - try: - all_quota = rest_client.account[self.user_id].quota.get()[1] - except HTTPError: - logging.exception("Unable to retrieve quota information from the accountpage") - self.user_scratch_storage = 0 - else: - muk_quota = filter(lambda q: q['storage']['name'] == self.muk.storage_name, all_quota) - if muk_quota: - self.user_scratch_quota = muk_quota[0]['hard'] * 1024 - else: - self.user_scratch_quota = 250 * 1024 * 1024 * 1024 - - self.scratch = self.gpfs.get_filesystem_info(self.muk.scratch_name) - - def pickle_path(self): - return self._scratch_path() - - def _scratch_path(self): - """Determines the path (relative to the scratch mount point) - - For a user with ID vscXYZUV this becomes users/vscXYZ/vscXYZUV. Note that the 'user' dir on scratch is - different, that is there to ensure the home dir symlink tree can be present on all nodes. - - @returns: string representing the relative path for this user. - """ - path = os.path.join(self.scratch['defaultMountPoint'], 'users', self.user_id[:-2], self.user_id) - return path - - def create_scratch_fileset(self): - """Create a fileset for the user on the scratch filesystem. - - - creates the fileset if it does not already exist - - sets the (fixed) quota on this fileset - - no user quota on scratch! only per-fileset quota - """ - self.gpfs.list_filesets() - - fileset_name = self.user_id - path = self._scratch_path() - - if not self.gpfs.get_fileset_info(self.muk.scratch_name, fileset_name): - logging.info("Creating new fileset on Muk scratch with name %s and path %s" % (fileset_name, path)) - base_dir_hierarchy = os.path.dirname(path) - self.gpfs.make_dir(base_dir_hierarchy) - self.gpfs.make_fileset(path, fileset_name) - else: - logging.info("Fileset %s already exists for user %s ... not doing anything." % (fileset_name, self.user_id)) - - self.gpfs.set_fileset_quota(self.user_scratch_quota, path, fileset_name) - - # We will always populate the scratch directory of the user as if it's his home directory - # In this way, if the user moves to home on scratch, everything will be up to date and in place. - - def populate_scratch_fallback(self): - """The scratch fileset is populated with the - - - ssh keys, - - a clean .bashrc script, - - a clean .bash_profile. - - The user can then always log in to the scratch, should the synchronisation fail to detect - a valid NFS mount point and avoid setting home on Muk. - """ - path = self._scratch_path() - self.gpfs.populate_home_dir(int(self.account.vsc_id_number), - int(self.usergroup.vsc_id_number), - path, - [p.pubkey for p in self.pubkeys]) - - def create_home_dir(self): - """Create the symlink to the real user's home dir that is - - - mounted somewhere over NFS - - has an AFM cache covering the real NFS mount - - sits on scratch (as indicated by the LDAP attribute). - """ - source = self.account.home_directory - base_home_dir_hierarchy = os.path.dirname(source.rstrip('/')) - target = None - - if 'VSC_MUK_SCRATCH' in [s.storage.name for s in self.home_on_scratch]: - logging.info("User %s has his home on Muk scratch" % (self.account.vsc_id)) - target = self._scratch_path() - elif 'VSC_MUK_AFM' in [s.storage.name for s in self.home_on_scratch]: - logging.info("User %s has his home on Muk AFM" % (self.user_id)) - target = self.muk.user_afm_home_mount(self.account.vsc_id, self.person.institute['site']) - - if target is None: - # This is the default case - target = self.muk.user_nfs_home_mount(self.account.vsc_id, self.person.institute['site']) - - self.gpfs.ignorerealpathmismatch = True - self.gpfs.make_dir(base_home_dir_hierarchy) - try: - os.symlink(target, source) # since it's just a link pointing to places that need not exist on the sync host - except OSError as err: - if err.errno not in [errno.EEXIST]: - raise - else: - logging.info("Symlink from %s to %s already exists" % (source, target)) - self.gpfs.ignorerealpathmismatch = False - - def cleanup_home_dir(self): - """Remove the symlink to the home dir for the user.""" - source = self.account.home_directory - - if self.gpfs.is_symlink(source): - os.unlink(source) - logging.info("Removed the symbolic link %s" % (source,)) - else: - logging.error("Home dir cleanup wanted to remove a non-symlink %s") - - def __setattr__(self, name, value): - """Override the setting of an attribute: - - - dry_run: set this here and in the gpfs and posix instance fields. - - otherwise, call super's __setattr__() - """ - - if name == 'dry_run': - self.gpfs.dry_run = value - self.posix.dry_run = value - - super(MukAccountpageUser, self).__setattr__(name, value) - cluster_user_pickle_location_map = { 'kyukon': VscTier2AccountpageUser, - 'muk': MukAccountpageUser, } cluster_user_pickle_store_map = { 'kyukon': 'VSC_SCRATCH_KYUKON', - 'muk': 'VSC_SCRATCH_MUK', } diff --git a/lib/vsc/administration/vo.py b/lib/vsc/administration/vo.py index 5f941821..1c660aee 100644 --- a/lib/vsc/administration/vo.py +++ b/lib/vsc/administration/vo.py @@ -31,16 +31,26 @@ from vsc.accountpage.wrappers import mkVo, mkVscVoSizeQuota, mkVscAccount from vsc.administration.tools import create_stat_directory from vsc.administration.user import VscTier2AccountpageUser, UserStatusUpdateError -from vsc.config.base import VSC, VscStorage, VSC_HOME, VSC_DATA, GENT_PRODUCTION_SCRATCH -from vsc.config.base import NEW, MODIFIED, MODIFY, ACTIVE, GENT +from vsc.config.base import VSC, VscStorage, VSC_HOME, VSC_DATA, VSC_DATA_SHARED, GENT_PRODUCTION_SCRATCH +from vsc.config.base import NEW, MODIFIED, MODIFY, ACTIVE, GENT, DATA_KEY, SCRATCH_KEY from vsc.filesystem.gpfs import GpfsOperations, GpfsOperationError, PosixOperations from vsc.utils.missing import Monoid, MonoidDict +SHARED = 'SHARED' + class VoStatusUpdateError(Exception): pass +def whenHTTPErrorRaise(f, msg, **kwargs): + try: + return f(**kwargs) + except HTTPError as err: + logging.error("%s: %s", msg, err) + raise + + class VscAccountPageVo(object): """ A Vo that gets its own information from the accountpage through the REST API. @@ -51,13 +61,14 @@ def __init__(self, vo_id, rest_client): """ self.vo_id = vo_id self.rest_client = rest_client + self._vo_cache = None - # We immediately retrieve this information - try: - self.vo = mkVo((rest_client.vo[vo_id].get()[1])) - except HTTPError: - logging.error("Cannot get information from the account page") - raise + @property + def vo(self): + if not self._vo_cache: + self._vo_cache = mkVo(whenHTTPErrorRaise(self.rest_client.vo[self.vo_id].get, + "Could not get VO from accountpage for VO %s" % self.vo_id)[1]) + return self._vo_cache class VscTier2AccountpageVo(VscAccountPageVo): @@ -81,20 +92,59 @@ def __init__(self, vo_id, storage=None, rest_client=None): self.gpfs = GpfsOperations() self.posix = PosixOperations() - try: - all_quota = [mkVscVoSizeQuota(q) for q in rest_client.vo[self.vo.vsc_id].quota.get()[1]] - except HTTPError: - logging.exception("Unable to retrieve quota information") - # to avoid reducing the quota in case of issues with the account page, we will NOT - # set quota when they cannot be retrieved. - self.vo_data_quota = None - self.vo_scratch_quota = None - else: - institute_quota = filter(lambda q: q.storage['institute'] == self.vo.institute['site'], all_quota) - self.vo_data_quota = ([q.hard for q in institute_quota - if q.storage['storage_type'] in ('data',)] or - [self.storage[VSC_DATA].quota_vo])[0] # there can be only one :) - self.vo_scratch_quota = filter(lambda q: q.storage['storage_type'] in ('scratch',), institute_quota) + self._vo_data_quota_cache = None + self._vo_data_shared_quota_cache = None + self._vo_scratch_quota_cache = None + self._institute_quota_cache = None + + @property + def _institute_quota(self): + if not self._institute_quota_cache: + all_quota = [mkVscVoSizeQuota(q) for q in + whenHTTPErrorRaise(self.rest_client.vo[self.vo.vsc_id].quota.get, + "Could not get quotata from accountpage for VO %s" % self.vo.vsc_id)[1] + ] + self._institute_quota_cache = [q for q in all_quota if q.storage['institute'] == self.vo.institute['site']] + return self._institute_quota_cache + + def _get_institute_data_quota(self): + return [q for q in self._institute_quota if q.storage['storage_type'] == DATA_KEY] + + def _get_institute_non_shared_data_quota(self): + return [q.hard for q in self._get_institute_data_quota() if not q.storage['name'].endswith(SHARED)] + + def _get_institute_shared_data_quota(self): + return [q.hard for q in self._get_institute_data_quota() if q.storage['name'].endswith(SHARED)] + + @property + def vo_data_quota(self): + if not self._vo_data_quota_cache: + self._vo_data_quota_cache = self._get_institute_non_shared_data_quota() + if not self._vo_data_quota_cache: + self._vo_data_quota_cache = [self.storage[VSC_DATA].quota_vo] + + return self._vo_data_quota_cache[0] # there can be only one + + @property + def vo_data_shared_quota(self): + if not self._vo_data_shared_quota_cache: + try: + self._vo_data_shared_quota_cache = self._get_institute_shared_data_quota()[0] + except IndexError: + return None + return self._vo_data_shared_quota_cache + + @property + def vo_scratch_quota(self): + if not self._vo_scratch_quota_cache: + self._vo_scratch_quota_cache = [q for q in self._institute_quota + if q.storage['storage_type'] == SCRATCH_KEY] + + return self._vo_scratch_quota_cache + + @property + def data_sharing(self): + return self.vo_data_shared_quota is not None def members(self): """Return a list with all the VO members in it.""" @@ -118,6 +168,10 @@ def _data_path(self, mount_point="gpfs"): """Return the path to the VO data fileset on GPFS""" return self._get_path(VSC_DATA, mount_point) + def _data_shared_path(self, mount_point="gpfs"): + """Return the path the VO shared data fileset on GPFS""" + return self._get_path(VSC_DATA_SHARED, mount_point) + def _scratch_path(self, storage, mount_point="gpfs"): """Return the path to the VO scratch fileset on GPFS. @@ -126,7 +180,7 @@ def _scratch_path(self, storage, mount_point="gpfs"): """ return self._get_path(storage, mount_point) - def _create_fileset(self, filesystem_name, path, parent_fileset=None): + def _create_fileset(self, filesystem_name, path, parent_fileset=None, fileset_name=None): """Create a fileset for the VO on the data filesystem. - creates the fileset if it does not already exist @@ -135,7 +189,8 @@ def _create_fileset(self, filesystem_name, path, parent_fileset=None): The parent_fileset is used to support older (< 3.5.x) GPFS setups still present in our system """ self.gpfs.list_filesets() - fileset_name = self.vo.vsc_id + if not fileset_name: + fileset_name = self.vo.vsc_id if not self.gpfs.get_fileset_info(filesystem_name, fileset_name): logging.info("Creating new fileset on %s with name %s and path %s" % @@ -166,13 +221,27 @@ def _create_fileset(self, filesystem_name, path, parent_fileset=None): def create_data_fileset(self): """Create the VO's directory on the HPC data filesystem. Always set the quota.""" + path = self._data_path() try: - path = self._data_path() - self._create_fileset(self.storage[VSC_DATA].filesystem, path) + fs = self.storage[VSC_DATA].filesystem except AttributeError: logging.exception("Trying to access non-existent attribute 'filesystem' in the storage instance") except KeyError: logging.exception("Trying to access non-existent field %s in the storage dictionary" % (VSC_DATA,)) + self._create_fileset(fs, path) + + def create_data_shared_fileset(self): + """Create a VO directory for sharing data on the HPC data filesystem. Always set the quota.""" + path = self._data_shared_path() + try: + fs = self.storage[VSC_DATA_SHARED].filesystem + except AttributeError: + logging.exception("Trying to access non-existent attribute 'filesystem' in the storage instance") + except KeyError: + logging.exception("Trying to access non-existent field %s in the storage dictionary" % (VSC_DATA_SHARED,)) + self._create_fileset(fs, path, fileset_name=self.vo.vsc_id.replace('gvo', 'gvos')) + + # TODO: change group ownership of this fileset junction path to that of the autogroup corresponding with the VO def create_scratch_fileset(self, storage_name): """Create the VO's directory on the HPC data filesystem. Always set the quota.""" @@ -191,18 +260,20 @@ def _create_vo_dir(self, path): """Create a user owned directory on the GPFS.""" self.gpfs.make_dir(path) - def _set_quota(self, storage_name, path, quota): + def _set_quota(self, storage_name, path, quota, fileset_name=None): """Set FILESET quota on the FS for the VO fileset. @type quota: int @param quota: soft quota limit expressed in KiB """ + if not fileset_name: + fileset_name = self.vo.vsc_id try: # expressed in bytes, retrieved in KiB from the backend hard = quota * 1024 * self.storage[storage_name].data_replication_factor soft = int(hard * self.vsc.quota_soft_fraction) # LDAP information is expressed in KiB, GPFS wants bytes. - self.gpfs.set_fileset_quota(soft, path, self.vo_id, hard) + self.gpfs.set_fileset_quota(soft, path, fileset_name, hard) self.gpfs.set_fileset_grace(path, self.vsc.vo_storage_grace_time) # 7 days except GpfsOperationError: logging.exception("Unable to set quota on path %s" % (path)) @@ -215,6 +286,14 @@ def set_data_quota(self): else: self._set_quota(VSC_DATA, self._data_path(), 16 * 1024) + def set_data_shared_quota(self): + """Set FILESET quota on the data FS for the VO fileset.""" + if self.vo_data_shared_quota: + self._set_quota(VSC_DATA_SHARED, + self._data_shared_path(), + int(self.vo_data_shared_quota), + fileset_name=self.vo.vsc_id.replace("gvo", "gvos")) + def set_scratch_quota(self, storage_name): """Set FILESET quota on the scratch FS for the VO fileset.""" if self.vo_scratch_quota: @@ -414,6 +493,10 @@ def process_vos(options, vo_ids, storage_name, client, datestamp): vo.set_data_quota() update_vo_status(vo, client) + if storage_name in [VSC_DATA_SHARED] and vo_id not in VSC().institute_vos.values() and vo.data_sharing: + vo.create_data_shared_fileset() + vo.set_data_shared_quota() + if vo_id in (VSC().institute_vos[GENT],): logging.info("Not deploying default VO %s members" % (vo_id,)) continue diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 79685ae0..00000000 --- a/setup.cfg +++ /dev/null @@ -1,15 +0,0 @@ -[bdist_rpm] -requires = python-vsc-accountpage-clients >= 0.9.0 - python-vsc-base >= 2.4.16 - python-vsc-config >= 1.31.2 - python-vsc-filesystems >= 0.19 - python-vsc-ldap >= 1.1 - pytz - python-ldap - python-vsc-ldap-extension >= 1.3 - python-vsc-utils >= 1.4.4 - python-lockfile >= 0.9.1 - -[metadata] -description-file = README.md - diff --git a/setup.py b/setup.py index 91f94e04..57376567 100644 --- a/setup.py +++ b/setup.py @@ -24,15 +24,15 @@ from vsc.install.shared_setup import ag, jt PACKAGE = { - 'version': '1.0.10', + 'version': '1.1.0', 'author': [ag, jt], 'maintainer': [ag, jt], 'tests_require': ['mock'], 'makesetupcfg': False, # use setup.cfg provided to get pytz instead of python-pytz 'install_requires': [ - 'vsc-accountpage-clients >= 0.9.0', + 'vsc-accountpage-clients >= 0.9.1', 'vsc-base >= 2.4.16', - 'vsc-config >= 1.31.2', + 'vsc-config >= 1.35.3', 'vsc-filesystems >= 0.19', 'vsc-ldap >= 1.1', 'pytz', diff --git a/test/tools.py b/test/tools.py index a9b9fc79..5c64ae02 100644 --- a/test/tools.py +++ b/test/tools.py @@ -18,17 +18,10 @@ @author: Andy Georges (Ghent University) """ import mock -import os -import stat from collections import namedtuple -import vsc.filesystem.posix -import vsc.administration.tools - -from vsc.accountpage.wrappers import mkVscAccount -from vsc.administration.tools import create_stat_directory, cleanup_purgees - +from vsc.administration.tools import create_stat_directory from vsc.install.testing import TestCase @@ -143,51 +136,3 @@ def test_create_stat_dir_existing_override(self, mock_posix, mock_stat_s_imode, mock_os_stat.assert_called_with(test_path) self.assertFalse(mock_posix.make_dir.called) mock_posix.chmod.assert_called_with(test_permissions, test_path) - - -class PurgeesTest(TestCase): - """" - Testcases for everything related to purged users. - """ - - @mock.patch('vsc.administration.user.MukAccountpageUser', autospec=True) - @mock.patch('vsc.administration.tools.notify_reinstatement') - @mock.patch('vsc.accountpage.client.AccountpageClient', autospec=True) - def test_cleanup_purgees(self, mock_client, mock_notify_reinstatement, mock_accountpage_user): - """ - Test that we're selecting the correct people to remove from the purgees list - """ - test_current_users = {1: 'a', 2: 'b', 3: 'c', 4: 'd', 5: 'e'} - test_current_purgees = {8: 'a', 2: 'b', 4: 'c', 6: 'd', 7: 'e'} - test_account = mkVscAccount({ - u'broken': False, - u'create_timestamp': u'1970-01-01T00:00:00.197Z', - u'data_directory': u'/user/data/gent/vsc400/vsc40075', - u'email': u'foobar@ugent.be', - u'home_directory': u'/user/home/gent/vsc400/vsc40075', - u'login_shell': u'/bin/bash', - u'person': { - u'gecos': u'Foo Bar', - u'institute': {u'site': u'gent'}, - u'institute_login': u'foobar' - }, - u'research_field': [u'Bollocks', u'Pluto'], - u'scratch_directory': u'/user/scratch/gent/vsc400/vsc40075', - u'status': u'active', - u'vsc_id': u'vsc40075', - u'vsc_id_number': 2540075 - }) - - mock_notify_reinstatement.return_value = None - mock_accountpage_user.return_value = mock.MagicMock() - mock_accountpage_user.person = test_account.person - mock_accountpage_user.dry_run = False - mock_client.return_value = mock.MagicMock() - mock_client.group = mock.MagicMock() - mock_client.group["gt1_mukgraceusers"] = mock.MagicMock() - mock_client.group["gt1_mukgraceusers"].member = mock.MagicMock() - - purgees_undone = cleanup_purgees(test_current_users, test_current_purgees, mock_client, False) - - self.assertNotEqual(mock_notify_reinstatement.calls, []) - self.assertTrue(purgees_undone == 2) diff --git a/test/user.py b/test/user.py index fa4e818f..4b76872f 100644 --- a/test/user.py +++ b/test/user.py @@ -45,7 +45,11 @@ u'scratch_directory': u'/user/scratch/gent/vsc400/vsc40075', u'status': u'active', u'vsc_id': u'vsc40075', - u'vsc_id_number': 2540075 + u'vsc_id_number': 2540075, + u'expiry_date': u'2032-01-01', + u'home_on_scratch': False, + u'force_active': False, + } test_account_2 = { @@ -64,7 +68,10 @@ u'scratch_directory': u'/user/scratch/gent/vsc400/vsc40075', u'status': u'active', u'vsc_id': u'vsc40075', - u'vsc_id_number': 2540075 + u'vsc_id_number': 2540075, + u'expiry_date': u'2034-01-01', + u'home_on_scratch': False, + u'force_active': False, } test_usergroup_1 = { @@ -132,8 +139,10 @@ "institute": { "site": "gent" }, - "institute_login": "ageorges" - } + "institute_login": "ageorges", + }, + u'home_on_scratch': False, + u'force_active': False, }, "storage": { "institute": "gent", diff --git a/test/vo.py b/test/vo.py index d9081dbd..7d77d8f3 100644 --- a/test/vo.py +++ b/test/vo.py @@ -25,7 +25,7 @@ import vsc.administration.vo as vo -from vsc.config.base import VSC_DATA, VSC_HOME, GENT_PRODUCTION_SCRATCH +from vsc.config.base import VSC_DATA, VSC_HOME, GENT_PRODUCTION_SCRATCH, VSC_DATA_SHARED from vsc.install.testing import TestCase @@ -228,3 +228,47 @@ def test_process_gent_institute_vo(self, mock_storage, mock_client): mock_s_m_d_quota.assert_not_called() mock_cr_m_d_dir.assert_not_called() + + + @mock.patch('vsc.accountpage.client.AccountpageClient', autospec=True) + @patch('vsc.administration.vo.VscStorage', autospec=True) + def test_process_gent_institute_vo_data_share(self, mock_storage, mock_client): + + test_vo_id = "gvo03442" + Options = namedtuple("Options", ['dry_run']) + options = Options(dry_run=False) + + mc = mock_client.return_value + mc.vo = mock.MagicMock() + v = mock.MagicMock() + mc.vo[test_vo_id].get.return_value = v + + for storage_name in (VSC_DATA_SHARED,): + with mock.patch("vsc.administration.vo.VscTier2AccountpageVo.data_sharing", new_callable=mock.PropertyMock) as mock_data_sharing: + with mock.patch('vsc.administration.vo.update_vo_status') as mock_update_vo_status: + with mock.patch.object(vo.VscTier2AccountpageVo, 'create_scratch_fileset') as mock_cr_s_fileset: + with mock.patch.object(vo.VscTier2AccountpageVo, 'set_scratch_quota') as mock_s_s_quota: + with mock.patch.object(vo.VscTier2AccountpageVo, 'create_data_fileset') as mock_cr_d_fileset: + with mock.patch.object(vo.VscTier2AccountpageVo, 'set_data_quota') as mock_s_d_quota: + with mock.patch.object(vo.VscTier2AccountpageVo, 'create_data_shared_fileset') as mock_cr_d_shared_fileset: + with mock.patch.object(vo.VscTier2AccountpageVo, 'set_data_shared_quota') as mock_s_d_shared_quota: + with mock.patch.object(vo.VscTier2AccountpageVo, 'set_member_data_quota') as mock_s_m_d_quota: + with mock.patch.object(vo.VscTier2AccountpageVo, 'create_member_data_dir') as mock_cr_m_d_dir: + with mock.patch.object(vo.VscTier2AccountpageVo, 'set_member_scratch_quota') as mock_s_m_s_quota: + with mock.patch.object(vo.VscTier2AccountpageVo, 'create_member_scratch_dir') as mock_cr_m_s_dir: + + mock_data_sharing.return_value = True + + ok, errors = vo.process_vos(options, [test_vo_id], storage_name, mc, "99991231") + self.assertEqual(errors, {}) + + mock_cr_s_fileset.assert_not_called() + mock_s_s_quota.assert_not_called() + mock_cr_d_fileset.assert_not_called() + mock_s_d_quota.assert_not_called() + mock_cr_d_shared_fileset.assert_called() + mock_s_d_shared_quota.assert_called() + mock_update_vo_status.assert_not_called() + + mock_s_m_d_quota.assert_not_called() + mock_cr_m_d_dir.assert_not_called()