diff --git a/bin/get_overview_users.py b/bin/get_overview_users.py deleted file mode 100644 index a0b2031e..00000000 --- a/bin/get_overview_users.py +++ /dev/null @@ -1,175 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2012-2017 Ghent University -# -# This file is part of vsc-administration, -# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), -# with support of Ghent University (http://ugent.be/hpc), -# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), -# the Flemish Research Foundation (FWO) (http://www.fwo.be/en) -# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). -# -# https://github.com/hpcugent/vsc-administration -# -# All rights reserved. -# -""" -This script produces an overview of the HPC users. - -@author: Andy Georges -@author: Wouter Depypere -""" - -from collections import namedtuple -from sqlalchemy import MetaData, Table, create_engine, select -from urllib2 import HTTPError - -from vsc.accountpage.client import AccountpageClient -from vsc.accountpage.wrappers import mkVscPerson -from vsc.config.base import GENT -from vsc.config.options import VscOptions -from vsc.ldap.configuration import UGentLdapConfiguration -from vsc.ldap.filters import LdapFilter -from vsc.ldap.utils import LdapQuery -from vsc.utils import fancylogger -from vsc.utils.missing import Monoid, MonoidDict -from vsc.utils.script_tools import ExtendedSimpleOption - -User = namedtuple('User', ['vscid', 'ugentid', 'active', 'employee', 'student']) - -log = fancylogger.getLogger(__name__) -fancylogger.logToScreen(True) -fancylogger.logLevelInfo() - -CONFIG_FILE = '/etc/vsc_conf.cfg' -PASSWD_FILE = '/etc/vsc_passwd.cfg' -DATABASE_NAME = "hpccollector" -DATABASE_USER = "hpccollector" - - -def get_hpc_collector_users(db, members): - """Get the users from UGent in the HPC collector database.""" - users = [ - User(vscid=u, ugentid=None, active=a, employee=False, student=False) for (u, _, a) in - db.execute(select([members.c.uid, members.c.active]).where(members.c.inst == GENT)).fetchall() - ] - log.debug("Found the following users in the HPC collector DB: %s", users) - return users - - -def get_ugent_id(client, vscuid): - """Retrieve the UGent login from the account page""" - try: - person = mkVscPerson(**client.account[vscuid].person.get()[1]) - return person.institute_login - except HTTPError: - log.warning("Cannot fetch information for user %s", vscuid) - return None - - -def ugent_status(opts, ldap_query, ugentid): - """Check the UGent object classes for this users and return a tuple.""" - ldap_filter = LdapFilter("uid=%s" % (ugentid)) - - users = ldap_query.user_filter_search(ldap_filter, ['objectClass']) - - if users: - object_classes = users[0]['objectClass'] - employee = object_classes.count('ugentEmployee') > 0 - student = object_classes.count('ugentStudent') > 0 - opts.log.debug("User with UGent ID %s is employee: %s, student: %s" % (ugentid, employee, student)) - return (employee, student) - else: - return (False, False) - - -class UGentLdapQuery(LdapQuery): - pass - - -def main(): - options = { - 'account_page_url': ('Base URL of the account page', None, 'store', 'https://account.vscentrum.be/django'), - 'access_token': ('OAuth2 token to access the account page REST API', None, 'store', None), - } - opts = ExtendedSimpleOption(options) - - global log - log = opts.log - - vsc_options = VscOptions(go_args=[], go_configfiles=[CONFIG_FILE, PASSWD_FILE]) - - client = AccountpageClient(token=opts.options.access_token) - - db_password = getattr(vsc_options.options, 'hpccollector_hpccollector') - db_engine = create_engine('postgresql://%s:%s@localhost/%s' % (DATABASE_USER, db_password, DATABASE_NAME)) - db_connection = db_engine.connect() - meta = MetaData() - - member = Table('member', meta, autoload=True, autoload_with=db_engine) - - users = get_hpc_collector_users(db_connection, member) - users = [u._replace(ugentid=get_ugent_id(client, u.vscid)) for u in users] - - ugent_ldap_query = UGentLdapQuery(UGentLdapConfiguration("collector")) # Initialise the LDAP connection - users = [u._replace(employee=employee, student=student) for u in users for (employee, student) in - [ugent_status(opts, ugent_ldap_query, u.ugentid)]] - - addm = Monoid(0, lambda x, y: x+y) - - student_type = (False, True) - employee_type = (True, False) - both_type = (True, True) - none_type = (False, False) - - user_types = MonoidDict(addm) - active_user_types = MonoidDict(addm) - inactive_user_types = MonoidDict(addm) - - active_users = 0 - inactive_users = 0 - - output = ["-" * 65] - output += ["%8s - %8s - %6s - %8s - %7s" % ("vscID", "UGentID", "Active", "Employee", "Student")] - output += ["-" * 65] - - for user in users: - output += ["%8s - %8s - %6s - %8s - %7s" % (user.vscid, user.ugentid, user.active, user.employee, user.student)] - - user_type = (user.employee, user.student) - user_types[user_type] = 1 - - if user.active: - active_users += 1 - active_user_types[user_type] = 1 - else: - inactive_users += 1 - inactive_user_types[user_type] = 1 - - output += ["-" * 65] - - template = "number of %s: %d" - output += [template % ("users", len(users))] - output += [template % ("active users", active_users)] - output += [template % ("inactive users", inactive_users)] - output += [""] - output += [template % ("(only) students", user_types[student_type])] - output += [template % ("(only) employees", user_types[employee_type])] - output += [template % ("people who are both employee as student", user_types[both_type])] - output += [template % ("people who are neither", user_types[none_type])] - output += [""] - output += [template % ("active students", active_user_types[student_type])] - output += [template % ("active employees", active_user_types[employee_type])] - output += [template % ("active people who are both employee as student", active_user_types[both_type])] - output += [template % ("active people who are neither", active_user_types[none_type])] - output += [""] - output += [template % ("inactive students", inactive_user_types[student_type])] - output += [template % ("inactive employees", inactive_user_types[employee_type])] - output += [template % ("inactive people who are both employee as student", inactive_user_types[both_type])] - output += [template % ("inactive people who are neither", inactive_user_types[none_type])] - output += ["-" * 65] - - print "\n".join(output) - -if __name__ == '__main__': - main() diff --git a/bin/sync_django_ldap.py b/bin/sync_django_ldap.py index 82aa7174..876b6144 100644 --- a/bin/sync_django_ldap.py +++ b/bin/sync_django_ldap.py @@ -24,26 +24,13 @@ from datetime import datetime -from ldap import LDAPError +from vsc.config.base import VSC_CONF_DEFAULT_FILENAME -import django -django.setup() +from vsc.accountpage.client import AccountpageClient -from django.conf import settings -from django.contrib.auth.models import Group as DGroup -from django.contrib.auth.models import User -from django.utils.timezone import utc - -from account.models import Account, Person, Pubkey -from group.models import Autogroup, Group, UserGroup, VirtualOrganisation, Membership, VoMembership -from host.models import Storage, Site -from quota.models import UserSizeQuota, VirtualOrganisationSizeQuota - -from vsc.config.base import GENT, ACTIVE, VSC_CONF_DEFAULT_FILENAME +from vsc.administration.ldapsync import LdapSyncer, ERROR from vsc.ldap.configuration import VscConfiguration -from vsc.ldap.entities import VscLdapUser, VscLdapGroup -from vsc.ldap.filters import CnFilter from vsc.ldap.timestamp import convert_timestamp, read_timestamp, write_timestamp from vsc.ldap.utils import LdapQuery from vsc.utils import fancylogger @@ -54,629 +41,25 @@ NAGIOS_CHECK_INTERVAL_THRESHOLD = 15 * 60 # 15 minutes SYNC_TIMESTAMP_FILENAME = "/var/cache/%s.timestamp" % (NAGIOS_HEADER) -ACCOUNT_WITHOUT_PUBLIC_KEYS_MAGIC_STRING = "THIS ACCOUNT HAS NO VALID PUBLIC KEYS" - fancylogger.setLogLevelInfo() fancylogger.logToScreen(True) _log = fancylogger.getLogger(NAGIOS_HEADER) -DONE = 'done' -NEW = 'new' -UPDATED = 'updated' -ERROR = 'error' - - -def add_or_update(VscLdapKlass, cn, ldap_attributes, dry_run): - """ - Perform the update in LDAP for the given vsc.ldap.entitities class, cn and the ldap attributes. - - @return: NEW, UPDATED or ERROR, depending on the operation and its result. - """ - ldap_entries = VscLdapKlass.lookup(CnFilter(cn)) - if not ldap_entries: - # add the entry - _log.debug("add new entry %s %s with the attributes %s", VscLdapKlass.__name__, cn, ldap_attributes) - - if not dry_run: - try: - entry = VscLdapKlass(cn) - entry.add(ldap_attributes) - _log.info("Added a new user %s to LDAP" % (cn,)) - except LDAPError: - _log.warning("Could not add %s %s to LDAP" % (VscLdapKlass.__name__, cn,)) - return ERROR - return NEW - else: - ldap_entries[0].status - _log.debug("update existing entry %s %s with the attributes %s -- old entry: %s", - VscLdapKlass.__name__, cn, ldap_attributes, ldap_entries[0].ldap_info) - - if not dry_run: - try: - ldap_entries[0].modify_ldap(ldap_attributes) - _log.info("Modified %s %s in LDAP" % (VscLdapKlass.__name__, cn,)) - except LDAPError: - _log.warning("Could not add %s %s to LDAP" % (VscLdapKlass.__name__, cn,)) - return ERROR - return UPDATED - - -def sync_altered_pubkeys(last, now, processed_accounts=None): - """ - Remove obsolete public keys from the LDAP and add new public keys to the LDAP. - """ - changed_pubkeys = Pubkey.objects.filter(modify_timestamp__range=[last, now]) - - pubkeys = { - UPDATED: set(), - DONE: set(), - ERROR: set(), - } - - new_pubkeys = [p for p in changed_pubkeys if not p.deleted] - deleted_pubkeys = [p for p in changed_pubkeys if p.deleted] - - _log.warning("Deleted pubkeys %s" % (deleted_pubkeys,)) - _log.debug("New pubkeys %s", new_pubkeys) - - for p in changed_pubkeys: - - if not p.user: - # This should NOT happen - _log.error("Key %d had no associated user any more: %s" % (p.pk, p)) - continue - - try: - account = p.user.account - except User.DoesNotExist: - _log.warning("No user found for the given public key %d" % (p.pk,)) - continue - except Account.DoesNotExist: - _log.warning("No account for the user %s corresponding to the public key %d" % (p.user.username, p.pk)) - continue - - if account in processed_accounts[NEW] or account in processed_accounts[UPDATED]: - _log.info("Account %s was already processed and has the new set of public keys" % (account.vsc_id,)) - pubkeys[DONE].add(p) - continue - - try: - pks = Pubkey.objects.filter(user=p.user, deleted=False) - ldap_user = VscLdapUser(account.vsc_id) - ldap_user.pubkey = [p.pubkey for p in pks] - processed_accounts[UPDATED].add(account) - pubkeys[UPDATED].add(p) - except Exception: - _log.warning("Cannot add pubkey for account %s to LDAP" % (account.vsc_id,)) - pubkeys[ERROR].add(p) - - return pubkeys - - -def sync_altered_users(last, now, processed_accounts): - """ - The only thing that can be changed is the email address, but we should sync that too. - """ - changed_users = User.objects.filter(modify_timestamp__range=[last, now]) - - _log.info("Changed users: %s" % ([u.username for u in changed_users])) - - users = { - UPDATED: set(), - DONE: set(), - ERROR: set(), - } - - for u in changed_users: - - if u.account in processed_accounts[NEW] or u in processed_accounts[UPDATED]: - _log.info("Account %s was already processed and has the new email address" % (u.account.vsc_id,)) - users[DONE].add(u) - continue - - try: - ldap_user = VscLdapUser(u.account.vsc_id) - ldap_user.mail = u.email - processed_accounts[UPDATED].add(u.account) - users[UPDATED].add(u) - except Exception: - _log.warning("Cannot change email address to %s for %s in LDAP" % (u.email, u.account.vsc_id)) - users[ERROR].add(u) - - return users - - -def sync_altered_accounts(last, now, dry_run=True): - """ - Add new users to the LDAP and update altered users. This does not include usergroups. - - @type last: datetime - @type now: datetime - @return: tuple (new, updated, error) that indicates what accounts were new, changed or could not be altered. - """ - changed_accounts = Account.objects.filter(modify_timestamp__range=[last, now]) - - accounts = { - NEW: set(), - UPDATED: set(), - ERROR: set(), - } - - sync_accounts = list(changed_accounts) - - _log.info("Found %d modified accounts in the range %s until %s" % (len(sync_accounts), - last.strftime("%Y%m%d%H%M%SZ"), - now.strftime("%Y%m%d%H%M%SZ"))) - _log.debug("Modified accounts: %s", [a.vsc_id for a in sync_accounts]) - - for account in sync_accounts: - - try: - home_storage = Storage.objects.get(storage_type=settings.HOME, institute=account.user.person.institute) - home_quota = UserSizeQuota.objects.get(user=account, storage=home_storage).hard - except UserSizeQuota.DoesNotExist: - home_quota = 0 - _log.warning("Could not find quota information for %s on %s, setting to 0" % (account.vsc_id, home_storage.name)) - except Storage.DoesNotExist: - home_quota = 0 - _log.warning("No home storage for institute %s defined, setting quota to 0" % (account.user.person.institute,)) - except User.DoesNotExist: - _log.error("No corresponding User for account %s" % (account.vsc_id,)) - continue - except Person.DoesNotExist: - _log.error("No corresponding Person for account %s" % (account.vsc_id,)) - continue - - try: - data_storage = Storage.objects.get(storage_type=settings.DATA, institute=account.user.person.institute) - data_quota_ = UserSizeQuota.objects.filter(user=account, storage=data_storage) - if len(list(data_quota_)) > 1: - # this is the UGent case; we need to further distinguish between our various filesets, in - # this case the vscxyz fileset - data_quota = data_quota_.get(fileset=account.vsc_id[:6]).hard - else: - data_quota = data_quota_[0].hard - except (UserSizeQuota.DoesNotExist, IndexError): - data_quota = 0 - _log.warning("Could not find quota information for %s on %s, setting to 0" % (account.vsc_id, data_storage.name)) - except Storage.DoesNotExist: - data_quota = 0 - _log.warning("No data storage for institute %s defined, setting quota to 0" % (account.user.person.institute,)) - - try: - scratch_storage = Storage.objects.filter(storage_type=settings.SCRATCH, institute=account.user.person.institute) - if not scratch_storage: - raise Storage.DoesNotExist("No scratch storage for institute %s" % (account.user.person.institute,)) - - if account.user.person.institute in (Site.objects.get(site=GENT),): - scratch_storage = scratch_storage.filter(name='VSC_SCRATCH_DELCATTY')[0] # TODO: This can be ignored once we go to sync from django, skipping the LDAP completely - else: - scratch_storage = scratch_storage[0] - - scratch_quota_ = UserSizeQuota.objects.filter(user=account, storage=scratch_storage) # take the first one - if len(list(scratch_quota_)) > 1: - # this is the UGent case; we need to further distinguish between our various filesets, in - # this case the vscxyz fileset - scratch_quota = scratch_quota_.get(fileset=account.vsc_id[:6]).hard - else: - scratch_quota = scratch_quota_[0].hard - except (UserSizeQuota.DoesNotExist, IndexError): - scratch_quota = 0 - _log.warning("Could not find quota information for %s on %s, setting to 0" % (account.vsc_id, scratch_storage.name)) - except Storage.DoesNotExist: - scratch_quota = 0 - _log.warning("No scratch storage for institute %s defined, setting quota to 0" % (account.user.person.institute,)) - - try: - try: - gecos = str(account.user.person.gecos) - except UnicodeEncodeError: - gecos = account.user.person.gecos.encode('ascii', 'ignore') - _log.warning("Converting unicode to ascii for gecos resulting in %s", gecos) - - public_keys = [str(p.pubkey) for p in Pubkey.objects.filter(user=account.user, deleted=False)] - if not public_keys: - public_keys = [ACCOUNT_WITHOUT_PUBLIC_KEYS_MAGIC_STRING] - - ldap_attributes = { - 'cn': str(account.vsc_id), - 'uidNumber': ["%s" % (account.vsc_id_number,)], - 'gecos': [gecos], - 'mail': [str(account.user.email)], - 'institute': [str(account.user.person.institute.site)], - 'instituteLogin': [str(account.user.person.institute_login)], - 'uid': [str(account.vsc_id)], - 'homeDirectory': [str(account.home_directory)], - 'dataDirectory': [str(account.data_directory)], - 'scratchDirectory': [str(account.scratch_directory)], - 'homeQuota': ["%d" % (home_quota,)], - 'dataQuota': ["%d" % (data_quota,)], - 'scratchQuota': ["%d" % (scratch_quota,)], - 'pubkey': public_keys, - 'gidNumber': ["%s" % (account.usergroup.vsc_id_number,)], - 'loginShell': [str(account.login_shell)], - # 'mukHomeOnScratch': ["FALSE"], # FIXME, see #37 - 'researchField': ["unknown"], - 'status': [str(account.status)], - } - except UserGroup.DoesNotExist: - _log.error("No corresponding UserGroup for user %s" % (account.vsc_id,)) - continue - - result = add_or_update(VscLdapUser, account.vsc_id, ldap_attributes, dry_run) - accounts[result].add(account) - - return accounts - - -def sync_altered_user_quota(last, now, altered_accounts): - """ - Check for users who have altered quota and sync these to the LDAP. - - @type last: datetime - @type now: datetime - @return: tuple (new, updated, error) that indicates what accounts were new, changed or could not be altered. - """ - - changed_quota = UserSizeQuota.objects.filter(modify_timestamp__range=[last, now]) - - _log.info("Found %d modified quota in the range %s until %s" % (len(changed_quota), - last.strftime("%Y%m%d%H%M%SZ"), - now.strftime("%Y%m%d%H%M%SZ"))) - quotas = { - NEW: set(), - UPDATED: set(), - ERROR: set(), - DONE: set(), - } - - for quota in changed_quota: - account = quota.user - if account in altered_accounts[NEW] or account in altered_accounts[UPDATED]: - _log.info("Quota %s was already processed with account %s" % (quota, account.vsc_id)) - quotas[DONE].add(quota) - continue - - try: - ldap_user = VscLdapUser(account.vsc_id) - ldap_user.status - if quota.storage.storage_type in (settings.HOME,): - ldap_user.homeQuota = "%d" % (quota.hard,) - quotas[UPDATED].add(quota) - elif quota.storage.storage_type in (settings.DATA,): - ldap_user.dataQuota = "%d" % (quota.hard,) - quotas[UPDATED].add(quota) - elif quota.storage.storage_type in (settings.SCRATCH,): - ldap_user.scratchQuota = "%d" % (quota.hard,) - quotas[UPDATED].add(quota) - else: - _log.warning("Cannot sync quota to LDAP (storage type %s unknown)" % (quota.storage.storage_type,)) - - except Exception: - _log.warning("Cannot update quota %s" % (quota,)) - quotas[ERROR].add(quota) - - return quotas - - -def sync_altered_user_groups(last, now, dry_run=True): - """ - Add new usergroups to the LDAP and update altered usergroups. - - @type last: datetime - @type now: datetime - @return: tuple (new, updated, error) that indicates what usergroups were new, changed or could not be altered. - """ - - changed_usergroups = UserGroup.objects.filter(modify_timestamp__range=[last, now]) - - _log.info("Found %d modified usergroups in the range %s until %s" % (len(changed_usergroups), - last.strftime("%Y%m%d%H%M%SZ"), - now.strftime("%Y%m%d%H%M%SZ"))) - - _log.debug("Modified usergroups: %s", [g.vsc_id for g in changed_usergroups]) - - groups = { - NEW: set(), - UPDATED: set(), - ERROR: set(), - } - - for usergroup in changed_usergroups: - - ldap_attributes = { - 'cn': str(usergroup.vsc_id), - 'institute': [str(usergroup.institute.site)], - 'gidNumber': ["%d" % (usergroup.vsc_id_number,)], - 'moderator': [str(usergroup.vsc_id)], # a fixed single moderator! - 'memberUid': [str(usergroup.vsc_id)], # a single member - 'status': [str(usergroup.status)], - } - - result = add_or_update(VscLdapGroup, usergroup.vsc_id, ldap_attributes, dry_run) - groups[result].add(usergroup) - - return groups - - -def sync_altered_autogroups(dry_run=True): - - changed_autogroups = Autogroup.objects.all() # we always sync autogroups since we cannot know beforehand if their membership list changed - - _log.info("Found %d autogroups" % (len(changed_autogroups),)) - _log.debug("Autogroups: %s", [a.vsc_id for a in changed_autogroups]) - - groups = { - NEW: set(), - UPDATED: set(), - ERROR: set(), - } - - for autogroup in changed_autogroups: - - ldap_attributes = { - 'cn': str(autogroup.vsc_id), - 'institute': [str(autogroup.institute.site)], - 'gidNumber': ["%d" % (autogroup.vsc_id_number,)], - 'moderator': [str(u.account.vsc_id) for u in DGroup.objects.get(name='administrator_%s' % (autogroup.institute.site,)).user_set.all()], - 'memberUid': [str(a.vsc_id) for a in autogroup.get_members()], - 'status': [str(autogroup.status)], - 'autogroup': [str(s.vsc_id) for s in autogroup.sources.all()], - } - - _log.debug("Proposed changes for autogroup %s: %s", autogroup.vsc_id, ldap_attributes) - - result = add_or_update(VscLdapGroup, autogroup.vsc_id, ldap_attributes, dry_run) - groups[result].add(autogroup) - - return groups - - -def sync_altered_group_membership(last, now, processed_groups): - """ - Synchronise the memberships for groups when users are added/removed. - """ - changed_members = Membership.objects.filter(modify_timestamp__range=[last, now]) - - _log.info("Found %d modified members in the range %s until %s" % (len(changed_members), - last.strftime("%Y%m%d%H%M%SZ"), - now.strftime("%Y%m%d%H%M%SZ"))) - _log.debug("Modified members: %s", [m.account.vsc_id for m in changed_members]) - - members = { - NEW: set(), - UPDATED: set(), - ERROR: set(), - DONE: set(), - } - - newly_processed_groups = set() - for member in changed_members: - - if member.group in processed_groups[NEW] \ - or member.group in processed_groups[UPDATED] \ - or member.group in newly_processed_groups: - _log.info("Member %s was already processed with group %s" % (member.account.vsc_id, member.group)) - members[DONE].add(member) - continue - - try: - ldap_group = VscLdapGroup(member.group.vsc_id) - ldap_group.status - ldap_group.memberUid = [str(m.account.vsc_id) for m in Membership.objects.filter(group=member.group)] - except Exception: - _log.warning("Cannot add member %s to group %s" % (member.account.vsc_id, member.group.vsc_id)) - members[ERROR].add(member) - else: - newly_processed_groups.add(member.group) - members[UPDATED].add(member) - _log.info("Processed group %s member %s" % (member.group.vsc_id, member.account.vsc_id)) - - return members - - -def sync_altered_groups(last, now, dry_run=True): - """ - Synchronise altered groups back to LDAP. - """ - changed_groups = Group.objects.filter(modify_timestamp__range=[last, now]) - - _log.info("Found %d modified groups in the range %s until %s" % (len(changed_groups), - last.strftime("%Y%m%d%H%M%SZ"), - now.strftime("%Y%m%d%H%M%SZ"))) - _log.debug("Modified groups: %s", [g.vsc_id for g in changed_groups]) - groups = { - NEW: set(), - UPDATED: set(), - ERROR: set(), - } - for group in changed_groups: - - ldap_attributes = { - 'cn': str(group.vsc_id), - 'institute': [str(group.institute.site)], - 'gidNumber': ["%d" % (group.vsc_id_number,)], - 'moderator': [str(m.account.vsc_id) for m in Membership.objects.filter(moderator=True, group=group)], - 'memberUid': [str(a.vsc_id) for a in group.get_members()], - 'status': [str(group.status)], - } - - _log.debug("Proposed changes for group %s: %s", group.vsc_id, ldap_attributes) - - result = add_or_update(VscLdapGroup, group.vsc_id, ldap_attributes, dry_run) - groups[result].add(group) - - return groups - - -def sync_altered_vo_membership(last, now, processed_vos): - """ - Synchronise the memberships for groups when users are added/removed. - """ - changed_members = VoMembership.objects.filter(modify_timestamp__range=[last, now]) - - _log.info("Found %d modified members in the range %s until %s" % (len(changed_members), - last.strftime("%Y%m%d%H%M%SZ"), - now.strftime("%Y%m%d%H%M%SZ"))) - _log.debug("Modified VO members: %s", [m.account.vsc_id for m in changed_members]) - members = { - NEW: set(), - UPDATED: set(), - ERROR: set(), - DONE: set(), - } - newly_processed_vos = set() - - for member in changed_members: - - if member.group in processed_vos[NEW] \ - or member.group in processed_vos[UPDATED] \ - or member.group in newly_processed_vos: - _log.info("Member %s membership was already processed with group %s" % (member.account.vsc_id, member.group)) - members[DONE].add(member) - continue - - try: - ldap_group = VscLdapGroup(member.group.vsc_id) - ldap_group.status - ldap_group.memberUid = [str(m.account.vsc_id) for m in VoMembership.objects.filter(group=member.group)] - except Exception: - _log.warning("Cannot add member %s to group %s" % (member.account.vsc_id, member.group.vsc_id)) - members[ERROR].add(member) - else: - newly_processed_vos.add(member.group) - members[UPDATED].add(member) - _log.info("Processed group %s member %s" % (member.group.vsc_id, member.account.vsc_id)) - - return members - - -def sync_altered_VO(last, now, dry_run=True): - """ - Synchronise altered VOs back to the LDAP. - """ - changed_vos = VirtualOrganisation.objects.filter(modify_timestamp__range=[last, now]) - _log.info("Found %d modified vos in the range %s until %s" % (len(changed_vos), - last.strftime("%Y%m%d%H%M%SZ"), - now.strftime("%Y%m%d%H%M%SZ"))) - _log.debug("Modified VOs: %s", [v.vsc_id for v in changed_vos]) - - vos = { - NEW: set(), - UPDATED: set(), - ERROR: set(), - } - - for vo in changed_vos: - - try: - data_storage = Storage.objects.get(storage_type=settings.DATA, institute=vo.institute) - data_quota = VirtualOrganisationSizeQuota.objects.get(virtual_organisation=vo, storage=data_storage, fileset=vo.vsc_id).hard - except (VirtualOrganisationSizeQuota.DoesNotExist, IndexError): - data_quota = 0 - _log.warning("Could not find VO quota information for %s on %s, setting to 0" % (vo.vsc_id, data_storage.name)) - except Storage.DoesNotExist: - data_quota = 0 - _log.warning("No VO data storage for institute %s defined, setting quota to 0" % (vo.institute,)) - - try: - scratch_storage = Storage.objects.filter(storage_type=settings.SCRATCH, institute=vo.institute) - scratch_quota = VirtualOrganisationSizeQuota.objects.get(virtual_organisation=vo, storage=scratch_storage[0], fileset=vo.vsc_id).hard # take the first one - except (VirtualOrganisationSizeQuota.DoesNotExist, IndexError): - scratch_quota = 0 - _log.warning("Could not find VO quota information for %s on %s, setting to 0" % (vo.vsc_id, data_storage.name)) - except Storage.DoesNotExist: - scratch_quota = 0 - _log.warning("No VO scratch storage for institute %s defined, setting quota to 0" % (vo.institute,)) - - # Hack to deal with the anomaly that the VO admin actually 'belongs' to multiple VOs, only in the LDAP - # the moderator need not be a member - if vo.vsc_id in settings.VSC.institute_vos.values(): - moderators = ['vsc40024'] - else: - moderators = [str(m.account.vsc_id) for m in VoMembership.objects.filter(moderator=True, group=vo)] - - ldap_attributes = { - 'cn': str(vo.vsc_id), - 'institute': [str(vo.institute.site)], - 'gidNumber': ["%d" % (vo.vsc_id_number,)], - 'moderator': moderators, - 'memberUid': [str(m.account.vsc_id) for m in VoMembership.objects.filter(group=vo)], - 'status': [str(vo.status)], - 'fairshare': ["%d" % (vo.fairshare,)], - 'description': [str(vo.description)], - 'dataDirectory': [str(vo.data_path)], - 'scratchDirectory': [str(vo.scratch_path)], - 'dataQuota': [str(data_quota)], - 'scratchQuota': [str(scratch_quota)], - } - - _log.debug("Proposed changes for VO %s: %s", vo.vsc_id, ldap_attributes) - - result = add_or_update(VscLdapGroup, vo.vsc_id, ldap_attributes, dry_run) - vos[result].add(vo) - - return vos - - -def sync_altered_vo_quota(last, now, altered_vos): - """ - Sync the changed quota for the VO to the LDAP - """ - changed_quota = VirtualOrganisationSizeQuota.objects.filter( - virtual_organisation__status=ACTIVE, - modify_timestamp__range=[last, now]) - - _log.info("Found %d modified VO quota in the range %s until %s" % ( - len(changed_quota), last.strftime("%Y%m%d%H%M%SZ"), now.strftime("%Y%m%d%H%M%SZ")) - ) - quotas = { - NEW: set(), - UPDATED: set(), - ERROR: set(), - DONE: set(), - } - - for quota in changed_quota: - virtual_organisation = quota.virtual_organisation - if virtual_organisation in altered_vos[NEW] or virtual_organisation in altered_vos[UPDATED]: - _log.info("Quota %s was already processed with VO %s" % (quota, virtual_organisation.vsc_id)) - quotas[DONE].add(quota) - continue - - try: - ldap_group = VscLdapGroup(virtual_organisation.vsc_id) - ldap_group.status - if quota.storage.storage_type in (settings.HOME,): - ldap_group.homeQuota = "%d" % (quota.hard,) - quotas[UPDATED].add(quota) - elif quota.storage.storage_type in (settings.DATA,): - ldap_group.dataQuota = "%d" % (quota.hard,) - quotas[UPDATED].add(quota) - elif quota.storage.storage_type in (settings.SCRATCH,): - ldap_group.scratchQuota = "%d" % (quota.hard,) - quotas[UPDATED].add(quota) - else: - _log.warning("Cannot sync quota to LDAP (storage type %s unknown)" % (quota.storage.storage_type,)) - - except Exception: - _log.warning("Cannot update quota %s" % (quota,)) - quotas[ERROR].add(quota) - - return quotas - def main(): - now = datetime.utcnow().replace(tzinfo=utc) options = { 'nagios-check-interval-threshold': NAGIOS_CHECK_INTERVAL_THRESHOLD, 'start-timestamp': ("The timestamp form which to start, otherwise use the cached value", None, "store", None), + 'access_token': ('OAuth2 token identifying the user with the accountpage', None, 'store', None), } + # get access_token from conf file + ExtendedSimpleOption.CONFIGFILES_INIT = ['/etc/account_page.conf'] opts = ExtendedSimpleOption(options) stats = {} - LdapQuery(VscConfiguration(VSC_CONF_DEFAULT_FILENAME)) + # Creating this here because this is a singleton class + _ = LdapQuery(VscConfiguration(VSC_CONF_DEFAULT_FILENAME)) last_timestamp = opts.options.start_timestamp if not last_timestamp: @@ -684,8 +67,8 @@ def main(): last_timestamp = read_timestamp(SYNC_TIMESTAMP_FILENAME) except Exception: _log.warning("Something broke reading the timestamp from %s" % SYNC_TIMESTAMP_FILENAME) - last_timestamp = "201404230000Z" - _log.warning("We will resync from the beginning of the account page era, i.e. %s" % (last_timestamp,)) + last_timestamp = "201707230000Z" + _log.warning("We will resync from a while back : %s" % (last_timestamp,)) _log.info("Using timestamp %s" % (last_timestamp)) @@ -716,44 +99,19 @@ def main(): except OSError: _log.raiseException("Could not drop privileges") - last = datetime.strptime(last_timestamp, "%Y%m%d%H%M%SZ").replace(tzinfo=utc) - - altered_accounts = sync_altered_accounts(last, now, opts.options.dry_run) - altered_pubkeys = sync_altered_pubkeys(last, now, altered_accounts) - # altered_users = sync_altered_users(last, now, altered_accounts) # FIXME: no modification timestamps here :( - - _log.debug("Altered accounts: %s" % (altered_accounts,)) - _log.debug("Altered pubkeys: %s" % (altered_pubkeys,)) - # _log.debug("Altered users: %s" % (altered_users,)) - - altered_usergroups = sync_altered_user_groups(last, now, opts.options.dry_run) - altered_autogroups = sync_altered_autogroups(opts.options.dry_run) + client = AccountpageClient(token=opts.options.access_token) + syncer = LdapSyncer(client) + last = int((datetime.strptime(last_timestamp, "%Y%m%d%H%M%SZ") - datetime(1970, 1, 1)).total_seconds()) + altered_accounts = syncer.sync_altered_accounts(last, opts.options.dry_run) - altered_groups = sync_altered_groups(last, now, opts.options.dry_run) - altered_members = sync_altered_group_membership(last, now, altered_groups) - altered_vos = sync_altered_VO(last, now, opts.options.dry_run) - altered_vo_members = sync_altered_vo_membership(last, now, altered_vos) + _log.debug("Altered accounts: %s", altered_accounts) - altered_user_quota = sync_altered_user_quota(last, now, altered_accounts) - altered_vo_quota = sync_altered_vo_quota(last, now, altered_vos) + altered_groups = syncer.sync_altered_groups(last, opts.options.dry_run) - _log.debug("Altered usergroups: %s" % (altered_usergroups,)) - _log.debug("Altered autogroups: %s" % (altered_autogroups,)) - _log.debug("Altered groups: %s" % (altered_groups,)) - _log.debug("Altered members: %s" % (altered_members,)) - _log.debug("Altered VOs: %s" % (altered_vos,)) - _log.debug("Altered VO members: %s" % (altered_vo_members,)) - _log.debug("Altered user quota: %s" % (altered_user_quota,)) - _log.debug("Altered VO quota: %s" % (altered_vo_quota,)) + _log.debug("Altered groups: %s" % altered_groups) if not altered_accounts[ERROR] \ - and not altered_groups[ERROR] \ - and not altered_vos[ERROR] \ - and not altered_members[ERROR] \ - and not altered_vo_members[ERROR] \ - and not altered_usergroups[ERROR] \ - and not altered_user_quota[ERROR] \ - and not altered_vo_quota[ERROR]: + and not altered_groups[ERROR]: _log.info("Child process exiting correctly") sys.exit(0) else: @@ -762,12 +120,6 @@ def main(): ["%s: %s\n" % (k, v) for (k, v) in [ ("altered accounts", altered_accounts[ERROR]), ("altered groups", altered_groups[ERROR]), - ("altered vos", altered_vos[ERROR]), - ("altered members", altered_members[ERROR]), - ("altered vo_members", altered_vo_members[ERROR]), - ("altered usergroups", altered_usergroups[ERROR]), - ("altered user quota", altered_user_quota[ERROR]), - ("altered vo quota", altered_vo_quota[ERROR]), ]] )) sys.exit(-1) @@ -782,7 +134,7 @@ def main(): if not result: if not opts.options.start_timestamp: - (_, ldap_timestamp) = convert_timestamp(now) + (_, ldap_timestamp) = convert_timestamp(datetime.now()) if not opts.options.dry_run: write_timestamp(SYNC_TIMESTAMP_FILENAME, ldap_timestamp) else: diff --git a/lib/vsc/administration/ldapsync.py b/lib/vsc/administration/ldapsync.py new file mode 100644 index 00000000..b30ae7e1 --- /dev/null +++ b/lib/vsc/administration/ldapsync.py @@ -0,0 +1,202 @@ +# -*- coding: latin-1 -*- +# +# Copyright 2013-2017 Ghent University +# +# This file is part of vsc-administration, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# the Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/hpcugent/vsc-administration +# +# All rights reserved. +# +""" +This module contains tools to sync accountpage users to the vsc ldap +""" +from __future__ import absolute_import + +from urllib2 import HTTPError + +import pytz as timezone +from datetime import datetime + +import logging + +from ldap import LDAPError + +from vsc.accountpage.wrappers import mkVscAccount, mkUserGroup, mkGroup, mkVo + +from vsc.ldap.entities import VscLdapUser, VscLdapGroup + +from vsc.ldap.filters import CnFilter + +ACCOUNT_WITHOUT_PUBLIC_KEYS_MAGIC_STRING = "THIS ACCOUNT HAS NO VALID PUBLIC KEYS" + +DONE = 'done' +NEW = 'new' +UPDATED = 'updated' +ERROR = 'error' + + +class LdapSyncer(object): + """ + This class implements a system for syncing changes from the accountpage api + to the vsc ldap + """ + def __init__(self, client): + """ + Create an ldap syncer, requires a RestClient client to get the information from + (typically AccountpageClient) + """ + self.client = client + self.now = datetime.utcnow().replace(tzinfo=timezone.utc) + + def add_or_update(self, VscLdapKlass, cn, ldap_attributes, dry_run): + """ + Perform the update in LDAP for the given vsc.ldap.entitities class, cn and the ldap attributes. + + @return: NEW, UPDATED or ERROR, depending on the operation and its result. + """ + ldap_entries = VscLdapKlass.lookup(CnFilter(cn)) + if not ldap_entries: + # add the entry + logging.debug("add new entry %s %s with the attributes %s", VscLdapKlass.__name__, cn, ldap_attributes) + + if not dry_run: + try: + entry = VscLdapKlass(cn) + entry.add(ldap_attributes) + logging.info("Added a new user %s to LDAP" % (cn,)) + except LDAPError: + logging.warning("Could not add %s %s to LDAP" % (VscLdapKlass.__name__, cn,)) + return ERROR + return NEW + else: + ldap_entries[0].status + logging.debug("update existing entry %s %s with the attributes %s -- old entry: %s", + VscLdapKlass.__name__, cn, ldap_attributes, ldap_entries[0].ldap_info) + + if not dry_run: + try: + ldap_entries[0].modify_ldap(ldap_attributes) + logging.info("Modified %s %s in LDAP" % (VscLdapKlass.__name__, cn,)) + except LDAPError: + logging.warning("Could not add %s %s to LDAP" % (VscLdapKlass.__name__, cn,)) + return ERROR + return UPDATED + + def sync_altered_accounts(self, last, dry_run=True): + """ + Add new users to the LDAP and update altered users. This does not include usergroups. + + this does include pubkeys + @type last: datetime + @return: tuple (new, updated, error) that indicates what accounts were new, changed or could not be altered. + """ + sync_accounts = [mkVscAccount(a) for a in self.client.account.modified[last].get()[1]] + accounts = { + NEW: set(), + UPDATED: set(), + ERROR: set(), + } + + logging.info("Found %d modified accounts in the range %s until %s" % (len(sync_accounts), + datetime.fromtimestamp(last).strftime("%Y%m%d%H%M%SZ"), + self.now.strftime("%Y%m%d%H%M%SZ"))) + logging.debug("Modified accounts: %s", [a.vsc_id for a in sync_accounts]) + + for account in sync_accounts: + try: + usergroup = mkUserGroup(self.client.account[account.vsc_id].usergroup.get()[1]) + except HTTPError: + logging.error("No corresponding UserGroup for user %s" % (account.vsc_id,)) + continue + try: + gecos = str(account.person.gecos) + except UnicodeEncodeError: + gecos = account.person.gecos.encode('ascii', 'ignore') + logging.warning("Converting unicode to ascii for gecos resulting in %s", gecos) + logging.debug('fetching public key') + + public_keys = [str(x.pubkey) for x in self.client.get_public_keys(account.vsc_id)] + if not public_keys: + public_keys = [ACCOUNT_WITHOUT_PUBLIC_KEYS_MAGIC_STRING] + + ldap_attributes = { + 'cn': str(account.vsc_id), + 'uidNumber': ["%s" % (account.vsc_id_number,)], + 'gecos': [gecos], + 'mail': [str(account.email)], + 'institute': [str(account.person.institute['site'])], + 'instituteLogin': [str(account.person.institute_login)], + 'uid': [str(account.vsc_id)], + 'homeDirectory': [str(account.home_directory)], + 'dataDirectory': [str(account.data_directory)], + 'scratchDirectory': [str(account.scratch_directory)], + 'pubkey': public_keys, + 'gidNumber': [str(usergroup.vsc_id_number)], + 'loginShell': [str(account.login_shell)], + 'researchField': [str(account.research_field[0])], + 'status': [str(account.status)], + } + logging.debug('fetching quota') + quotas = self.client.account[account.vsc_id].quota.get()[1] + for quota in quotas: + for stype in ['home', 'data', 'scratch']: + # only gent sets filesets for vo's, so not gvo is user. (other institutes is empty or "None" + if quota['storage']['storage_type'] == stype and not quota['fileset'].startswith('gvo'): + ldap_attributes['%sQuota' % stype] = ["%d" % quota["hard"]] + + result = self.add_or_update(VscLdapUser, account.vsc_id, ldap_attributes, dry_run) + accounts[result].add(account.vsc_id) + + return accounts + + def sync_altered_groups(self, last, dry_run=True): + """ + Synchronise altered groups back to LDAP. + This also includes usergroups + """ + changed_groups = [mkGroup(a) for a in self.client.allgroups.modified[last].get()[1]] + + logging.info("Found %d modified groups in the range %s until %s" % (len(changed_groups), + datetime.fromtimestamp(last).strftime("%Y%m%d%H%M%SZ"), + self.now.strftime("%Y%m%d%H%M%SZ"))) + logging.debug("Modified groups: %s", [g.vsc_id for g in changed_groups]) + groups = { + NEW: set(), + UPDATED: set(), + ERROR: set(), + } + + for group in changed_groups: + vo = False + try: + vo = mkVo(self.client.vo[group.vsc_id].get()[1]) + except HTTPError as err: + # if a 404 occured, the group is not an VO, so we skip this. Otherwise something else went wrong. + if err.code != 404: + raise + ldap_attributes = { + 'cn': str(group.vsc_id), + 'institute': [str(group.institute['site'])], + 'gidNumber': ["%d" % (group.vsc_id_number,)], + 'moderator': [str(m) for m in group.moderators], + 'memberUid': [str(a) for a in group.members], + 'status': [str(group.status)], + } + if vo: + ldap_attributes['fairshare'] = ["%d" % (vo.fairshare,)] + ldap_attributes['description'] = [str(vo.description)] + ldap_attributes['dataDirectory'] = [str(vo.data_path)] + ldap_attributes['scratchDirectory'] = [str(vo.scratch_path)] + + logging.debug("Proposed changes for group %s: %s", group.vsc_id, ldap_attributes) + + result = self.add_or_update(VscLdapGroup, group.vsc_id, ldap_attributes, dry_run) + groups[result].add(group.vsc_id) + + return groups diff --git a/lib/vsc/administration/user.py b/lib/vsc/administration/user.py index 7c635138..bc2ccff5 100644 --- a/lib/vsc/administration/user.py +++ b/lib/vsc/administration/user.py @@ -27,8 +27,8 @@ from urllib2 import HTTPError from vsc.utils import fancylogger -from vsc.accountpage.wrappers import mkVscAccountPubkey, mkVscHomeOnScratch, mkUserGroup -from vsc.accountpage.wrappers import mkVscAccount +from vsc.accountpage.wrappers import mkVscAccountPubkey, mkVscHomeOnScratch +from vsc.accountpage.wrappers import mkVscAccount, mkUserGroup from vsc.accountpage.wrappers import mkGroup, mkVscUserSizeQuota from vsc.administration.tools import create_stat_directory from vsc.config.base import VSC, Muk, VscStorage, VSC_DATA, VSC_HOME, GENT_PRODUCTION_SCRATCH diff --git a/setup.py b/setup.py index a1ef181e..913202ec 100644 --- a/setup.py +++ b/setup.py @@ -23,19 +23,21 @@ from vsc.install.shared_setup import ag PACKAGE = { - 'version': '0.37.0', + 'version': '1.0.0', 'author': [ag], 'maintainer': [ag], 'tests_require': ['mock'], 'install_requires': [ - 'vsc-accountpage-clients >= 0.7', + 'vsc-accountpage-clients >= 0.9.0', 'vsc-base >= 2.4.16', 'vsc-config >= 1.31.2', 'vsc-filesystems >= 0.19', 'vsc-ldap >= 1.1', + 'python-ldap', 'vsc-ldap-extension >= 1.3', 'vsc-utils >= 1.4.4', 'lockfile >= 0.9.1', + 'pytz', # following dependencies are intentionally not declared until #11 is addressed #'vsc-postgres', #'django', diff --git a/test/ldapsync.py b/test/ldapsync.py new file mode 100644 index 00000000..181b55a2 --- /dev/null +++ b/test/ldapsync.py @@ -0,0 +1,118 @@ +# +# Copyright 2015-2017 Ghent University +# +# This file is part of vsc-administration, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# the Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/hpcugent/vsc-administration +# +# All rights reserved. +# +""" +Tests for vsc.administration.user + +@author: Andy Georges (Ghent University) +@author: Jens Timmerman (Ghent University) +""" +import mock + +from urllib2 import HTTPError + +import vsc + +from vsc.install.testing import TestCase +from vsc.accountpage.wrappers import mkVscAccountPubkey, mkVscAccount, mkGroup + +from vsc.administration.ldapsync import LdapSyncer, UPDATED +from vsc.ldap.entities import VscLdapUser, VscLdapGroup + +from .user import test_account_1, test_usergroup_1, test_pubkeys_1 + +test_quota = [ + { + "user": "vsc10018", + "storage": { + "institute": "brussel", + "name": "VSC_HOME", + "storage_type": "home" + }, + "fileset": "None", + "hard": 5242880 + } +] + +test_vo_1 = { + "vsc_id": "gvo00003", + "status": "active", + "vsc_id_number": 2640010, + "institute": { + "site": "gent" + }, + "fairshare": 100, + "data_path": "/user/data/gent/gvo000/gvo00003", + "scratch_path": "/user/scratch/gent/gvo000/gvo00003", + "description": "VO", + "members": [ + "vsc40075", + ], + "moderators": [ + "vsc40075" + ] +} + + +class LDAPSyncerTest(TestCase): + """ + Tests for the LDAP syncer that sync account page information to the vsc ldap. + """ + + @mock.patch.object(vsc.administration.ldapsync.LdapSyncer, 'add_or_update') + def test_sync_altered_accounts(self, mock_add_or_update): + """Test the sync_altered accounts function""" + mock_client = mock.MagicMock() + test_account = mkVscAccount(test_account_1) + mock_client.account[test_account.vsc_id] = mock.MagicMock() + mock_client.account.modified[1].get.return_value = (200, [test_account_1]) + mock_client.account[test_account.vsc_id].usergroup.get.return_value = (200, test_usergroup_1) + mock_client.get_public_keys.return_value = [mkVscAccountPubkey(p) for p in test_pubkeys_1] + mock_client.account[test_account.vsc_id].quota.get.return_value = (200, test_quota) + + mock_add_or_update.return_value = UPDATED + ldapsyncer = LdapSyncer(mock_client) + accounts = ldapsyncer.sync_altered_accounts(1) + self.assertEqual(accounts, {'error': set([]), 'new': set([]), 'updated': set([test_account.vsc_id])}) + ldap_attrs = {'status': ['active'], 'dataDirectory': ['/user/data/gent/vsc400/vsc40075'], 'cn': 'vsc40075', 'homeQuota': ['5242880'], 'loginShell': ['/bin/bash'], 'uidNumber': ['2540075'], 'gidNumber': ['2540075'], 'instituteLogin': ['foobar'], 'uid': ['vsc40075'], 'scratchDirectory': ['/user/scratch/gent/vsc400/vsc40075'], 'institute': ['gent'], 'researchField': ['Bollocks'], 'gecos': ['Foo Bar'], 'homeDirectory': ['/user/home/gent/vsc400/vsc40075'], 'mail': ['foobar@ugent.be'], 'pubkey': ['pubkey1', 'pubkey2']} + mock_add_or_update.assert_called_with(VscLdapUser, test_account.vsc_id, ldap_attrs, True) + + @mock.patch.object(vsc.administration.ldapsync.LdapSyncer, 'add_or_update') + def test_sync_altered_groups(self, mock_add_or_update): + """Test the sync_altered accounts function""" + mock_client = mock.MagicMock() + test_group = mkGroup(test_vo_1) + mock_client.allgroups.modified[1].get.return_value = (200, [test_vo_1]) + mock_client.vo[test_group.vsc_id].get.return_value = (200, test_vo_1) + + mock_add_or_update.return_value = UPDATED + ldapsyncer = LdapSyncer(mock_client) + groups = ldapsyncer.sync_altered_groups(1) + self.assertEqual(groups, {'error': set([]), 'new': set([]), 'updated': set([test_group.vsc_id])}) + + ldap_attrs = {'status': ['active'], 'scratchDirectory': ['/user/scratch/gent/gvo000/gvo00003'], + 'dataDirectory': ['/user/data/gent/gvo000/gvo00003'], 'cn': 'gvo00003', 'institute': ['gent'], + 'memberUid': ['vsc40075'], 'moderator': ['vsc40075'], 'gidNumber': ['2640010'], + 'fairshare': ['100'], 'description': ['VO']} + mock_add_or_update.assert_called_with(VscLdapGroup, test_group.vsc_id, ldap_attrs, True) + + # raise a 404 error on getting the VO to trigger a non vo group add instead of a vo add as above + test_group = mkGroup(test_usergroup_1) + mock_client.allgroups.modified[1].get.return_value = (200, [test_usergroup_1]) + mock_client.vo[test_group.vsc_id].get.side_effect = HTTPError("mock_url", 404, "Not Found", "mock_headers", None) + groups = ldapsyncer.sync_altered_groups(1) + self.assertEqual(groups, {'error': set([]), 'new': set([]), 'updated': set([test_group.vsc_id])}) + ldap_attrs = {'status': ['active'], 'cn': 'vsc40075', 'institute': ['gent'], 'memberUid': ['vsc40075'], + 'moderator': ['vsc40075'], 'gidNumber': ['2540075']} + mock_add_or_update.assert_called_with(VscLdapGroup, test_group.vsc_id, ldap_attrs, True) diff --git a/test/user.py b/test/user.py index 384f4360..fa4e818f 100644 --- a/test/user.py +++ b/test/user.py @@ -13,11 +13,11 @@ # All rights reserved. # """ -Tests for vsc.administration.vo +Tests for vsc.administration.user @author: Andy Georges (Ghent University) +@author: Jens Timmerman (Ghent University) """ -import logging import mock from collections import namedtuple @@ -25,7 +25,7 @@ import vsc.administration.user as user from vsc.accountpage.wrappers import mkVscAccount, mkVscHomeOnScratch, mkUserGroup, mkGroup -from vsc.accountpage.wrappers import mkVscAccountPubkey, mkVscUserSizeQuota +from vsc.accountpage.wrappers import mkVscAccountPubkey from vsc.config.base import VSC_DATA, VSC_HOME, VSC_SCRATCH_PHANPY, VSC_SCRATCH_DELCATTY from vsc.install.testing import TestCase @@ -445,7 +445,7 @@ def test_create_data_dir_tier2_user(self, @mock.patch.object(user.VscTier2AccountpageUser, '_grouping_scratch_path') @mock.patch.object(user.VscTier2AccountpageUser, '_create_grouping_fileset') @mock.patch.object(user.VscTier2AccountpageUser, '_create_user_dir') - def test_create_data_dir_tier2_user(self, + def test_create_data_dir_tier2_user_2(self, mock_create_user_dir, mock_create_grouping_fileset, mock_grouping_scratch_path,