From bcf6a424a0bb6cccb04a81fda7adf26161a48789 Mon Sep 17 00:00:00 2001 From: Jens Timmerman Date: Tue, 14 Jun 2016 16:28:55 +0200 Subject: [PATCH 01/26] this info is visible in the accountpage admin view --- bin/get_overview_users.py | 175 -------------------------------------- 1 file changed, 175 deletions(-) delete mode 100644 bin/get_overview_users.py diff --git a/bin/get_overview_users.py b/bin/get_overview_users.py deleted file mode 100644 index f9be9998..00000000 --- a/bin/get_overview_users.py +++ /dev/null @@ -1,175 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2012-2016 Ghent University -# -# This file is part of vsc-administration, -# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), -# with support of Ghent University (http://ugent.be/hpc), -# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), -# the Flemish Research Foundation (FWO) (http://www.fwo.be/en) -# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). -# -# https://github.com/hpcugent/vsc-administration -# -# All rights reserved. -# -""" -This script produces an overview of the HPC users. - -@author: Andy Georges -@author: Wouter Depypere -""" - -from collections import namedtuple -from sqlalchemy import MetaData, Table, create_engine, select -from urllib2 import HTTPError - -from vsc.accountpage.client import AccountpageClient -from vsc.accountpage.wrappers import mkVscPerson -from vsc.config.base import GENT -from vsc.config.options import VscOptions -from vsc.ldap.configuration import UGentLdapConfiguration -from vsc.ldap.filters import LdapFilter -from vsc.ldap.utils import LdapQuery -from vsc.utils import fancylogger -from vsc.utils.missing import Monoid, MonoidDict -from vsc.utils.script_tools import ExtendedSimpleOption - -User = namedtuple('User', ['vscid', 'ugentid', 'active', 'employee', 'student']) - -log = fancylogger.getLogger(__name__) -fancylogger.logToScreen(True) -fancylogger.logLevelInfo() - -CONFIG_FILE = '/etc/vsc_conf.cfg' -PASSWD_FILE = '/etc/vsc_passwd.cfg' -DATABASE_NAME = "hpccollector" -DATABASE_USER = "hpccollector" - - -def get_hpc_collector_users(db, members): - """Get the users from UGent in the HPC collector database.""" - users = [ - User(vscid=u, ugentid=None, active=a, employee=False, student=False) for (u, i, a) in - db.execute(select([members.c.uid, members.c.active]).where(members.c.inst == GENT)).fetchall() - ] - log.debug("Found the following users in the HPC collector DB: %s", users) - return users - - -def get_ugent_id(opts, client, vscuid): - """Retrieve the UGent login from the account page""" - try: - person = mkVscPerson(**client.account[vscuid].person.get()[1]) - return person.institute_login - except HTTPError: - log.warning("Cannot fetch information for user %s", vscuid) - return None - - -def ugent_status(opts, ldap_query, ugentid): - """Check the UGent object classes for this users and return a tuple.""" - ldap_filter = LdapFilter("uid=%s" % (ugentid)) - - users = ldap_query.user_filter_search(ldap_filter, ['objectClass']) - - if users: - object_classes = users[0]['objectClass'] - employee = object_classes.count('ugentEmployee') > 0 - student = object_classes.count('ugentStudent') > 0 - opts.log.debug("User with UGent ID %s is employee: %s, student: %s" % (ugentid, employee, student)) - return (employee, student) - else: - return (False, False) - - -class UGentLdapQuery(LdapQuery): - pass - - -def main(): - options = { - 'account_page_url': ('Base URL of the account page', None, 'store', 'https://account.vscentrum.be/django'), - 'access_token': ('OAuth2 token to access the account page REST API', None, 'store', None), - } - opts = ExtendedSimpleOption(options) - - global log - log = opts.log - - vsc_options = VscOptions(go_args=[], go_configfiles=[CONFIG_FILE, PASSWD_FILE]) - - client = AccountpageClient(token=opts.options.access_token) - - db_password = getattr(vsc_options.options, 'hpccollector_hpccollector') - db_engine = create_engine('postgresql://%s:%s@localhost/%s' % (DATABASE_USER, db_password, DATABASE_NAME)) - db_connection = db_engine.connect() - meta = MetaData() - - member = Table('member', meta, autoload=True, autoload_with=db_engine) - - users = get_hpc_collector_users(db_connection, member) - users = [u._replace(ugentid=get_ugent_id(opts, client, u.vscid)) for u in users] - - ugent_ldap_query = UGentLdapQuery(UGentLdapConfiguration("collector")) # Initialise the LDAP connection - users = [u._replace(employee=employee, student=student) for u in users for (employee, student) in - [ugent_status(opts, ugent_ldap_query, u.ugentid)]] - - addm = Monoid(0, lambda x, y: x+y) - - student_type = (False, True) - employee_type = (True, False) - both_type = (True, True) - none_type = (False, False) - - user_types = MonoidDict(addm) - active_user_types = MonoidDict(addm) - inactive_user_types = MonoidDict(addm) - - active_users = 0 - inactive_users = 0 - - output = ["-" * 65] - output += ["%8s - %8s - %6s - %8s - %7s" % ("vscID", "UGentID", "Active", "Employee", "Student")] - output += ["-" * 65] - - for user in users: - output += ["%8s - %8s - %6s - %8s - %7s" % (user.vscid, user.ugentid, user.active, user.employee, user.student)] - - user_type = (user.employee, user.student) - user_types[user_type] = 1 - - if user.active: - active_users += 1 - active_user_types[user_type] = 1 - else: - inactive_users += 1 - inactive_user_types[user_type] = 1 - - output += ["-" * 65] - - template = "number of %s: %d" - output += [template % ("users", len(users))] - output += [template % ("active users", active_users)] - output += [template % ("inactive users", inactive_users)] - output += [""] - output += [template % ("(only) students", user_types[student_type])] - output += [template % ("(only) employees", user_types[employee_type])] - output += [template % ("people who are both employee as student", user_types[both_type])] - output += [template % ("people who are neither", user_types[none_type])] - output += [""] - output += [template % ("active students", active_user_types[student_type])] - output += [template % ("active employees", active_user_types[employee_type])] - output += [template % ("active people who are both employee as student", active_user_types[both_type])] - output += [template % ("active people who are neither", active_user_types[none_type])] - output += [""] - output += [template % ("inactive students", inactive_user_types[student_type])] - output += [template % ("inactive employees", inactive_user_types[employee_type])] - output += [template % ("inactive people who are both employee as student", inactive_user_types[both_type])] - output += [template % ("inactive people who are neither", inactive_user_types[none_type])] - output += ["-" * 65] - - print "\n".join(output) - -if __name__ == '__main__': - main() From dfac2e26798314ac00de2b43b2453df9eb3c9e45 Mon Sep 17 00:00:00 2001 From: Jens Timmerman Date: Tue, 14 Jun 2016 17:51:00 +0200 Subject: [PATCH 02/26] started working on a switch from django models to rest api --- bin/sync_django_ldap.py | 867 +++++++++++++-------------------- lib/vsc/administration/user.py | 18 +- 2 files changed, 340 insertions(+), 545 deletions(-) diff --git a/bin/sync_django_ldap.py b/bin/sync_django_ldap.py index 75519713..2627f577 100644 --- a/bin/sync_django_ldap.py +++ b/bin/sync_django_ldap.py @@ -22,25 +22,13 @@ import pwd import sys -from datetime import datetime +from datetime import datetime, timezone from ldap import LDAPError - -import django -django.setup() - -from django.conf import settings -from django.contrib.auth.models import Group as DGroup -from django.contrib.auth.models import User -from django.utils.timezone import utc - -from account.models import Account, Person, Pubkey, MailList -from group.models import Autogroup, Group, UserGroup, VirtualOrganisation, Membership, VoMembership -from host.models import Storage, Site -from quota.models import UserSizeQuota, VirtualOrganisationSizeQuota - from vsc.config.base import GENT, ACTIVE, VSC_CONF_DEFAULT_FILENAME +from vsc.accountpage.client import AccountpageClient + from vsc.ldap.configuration import VscConfiguration from vsc.ldap.entities import VscLdapUser, VscLdapGroup from vsc.ldap.filters import CnFilter @@ -54,7 +42,7 @@ NAGIOS_CHECK_INTERVAL_THRESHOLD = 15 * 60 # 15 minutes SYNC_TIMESTAMP_FILENAME = "/var/cache/%s.timestamp" % (NAGIOS_HEADER) -ACCOUNT_WITHOUT_PUBLIC_KEYS_MAGIC_STRING="THIS ACCOUNT HAS NO VALID PUBLIC KEYS" +ACCOUNT_WITHOUT_PUBLIC_KEYS_MAGIC_STRING = "THIS ACCOUNT HAS NO VALID PUBLIC KEYS" fancylogger.setLogLevelInfo() fancylogger.logToScreen(True) @@ -66,605 +54,419 @@ ERROR = 'error' -def add_or_update(VscLdapKlass, cn, ldap_attributes, dry_run): - """ - Perform the update in LDAP for the given vsc.ldap.entitities class, cn and the ldap attributes. - - @return: NEW, UPDATED or ERROR, depending on the operation and its result. - """ - ldap_entries = VscLdapKlass.lookup(CnFilter(cn)) - if not ldap_entries: - # add the entry - _log.debug("add new entry %s %s with the attributes %s", VscLdapKlass.__name__, cn, ldap_attributes) - - if not dry_run: - try: - entry = VscLdapKlass(cn) - entry.add(ldap_attributes) - _log.info("Added a new user %s to LDAP" % (cn,)) - except LDAPError: - _log.warning("Could not add %s %s to LDAP" % (VscLdapKlass.__name__, cn,)) - return ERROR - return NEW - else: - ldap_entries[0].status - _log.debug("update existing entry %s %s with the attributes %s -- old entry: %s", - VscLdapKlass.__name__, cn, ldap_attributes, ldap_entries[0].ldap_info) - - if not dry_run: - try: - ldap_entries[0].modify_ldap(ldap_attributes) - _log.info("Modified %s %s in LDAP" % (VscLdapKlass.__name__, cn,)) - except LDAPError: - _log.warning("Could not add %s %s to LDAP" % (VscLdapKlass.__name__, cn,)) - return ERROR - return UPDATED - - -def sync_altered_pubkeys(last, now, processed_accounts=None, dry_run=True): - """ - Remove obsolete public keys from the LDAP and add new public keys to the LDAP. - """ - changed_pubkeys = Pubkey.objects.filter(modify_timestamp__range=[last, now]) - - pubkeys = { - UPDATED: set(), - DONE: set(), - ERROR: set(), - } - - new_pubkeys = [p for p in changed_pubkeys if not p.deleted] - deleted_pubkeys = [p for p in changed_pubkeys if p.deleted] - - _log.warning("Deleted pubkeys %s" % (deleted_pubkeys,)) - _log.debug("New pubkeys %s", new_pubkeys) - - for p in changed_pubkeys: - - if not p.user: - # This should NOT happen - _log.error("Key %d had no associated user any more: %s" % (p.pk, p)) - continue - - try: - account = p.user.account - except User.DoesNotExist: - _log.warning("No user found for the given public key %d" % (p.pk,)) - continue - except Account.DoesNotExist: - _log.warning("No account for the user %s corresponding to the public key %d" % (p.user.username, p.pk)) - continue - - if account in processed_accounts[NEW] or account in processed_accounts[UPDATED]: - _log.info("Account %s was already processed and has the new set of public keys" % (account.vsc_id,)) - pubkeys[DONE].add(p) - continue - - try: - pks = Pubkey.objects.filter(user=p.user, deleted=False) - ldap_user = VscLdapUser(account.vsc_id) - ldap_user.pubkey = [p.pubkey for p in pks] - processed_accounts[UPDATED].add(account) - pubkeys[UPDATED].add(p) - except Exception: - _log.warning("Cannot add pubkey for account %s to LDAP" % (account.vsc_id,)) - pubkeys[ERROR].add(p) - - return pubkeys - - -def sync_altered_users(last, now, processed_accounts, dry_run=True): +def class LdapSyncer(object): """ - The only thing that can be changed is the email address, but we should sync that too. + This class implements a system for syncing changes from the accountpage api + to the vsc ldap """ - changed_users = User.objects.filter(modify_timestamp__range=[last, now]) - - _log.info("Changed users: %s" % ([u.username for u in changed_users])) - - users = { - UPDATED: set(), - DONE: set(), - ERROR: set(), - } - - for u in changed_users: - - if u.account in processed_accounts[NEW] or u in processed_accounts[UPDATED]: - _log.info("Account %s was already processed and has the new email address" % (u.account.vsc_id,)) - users[DONE].add(u) - continue - - try: - ldap_user = VscLdapUser(u.account.vsc_id) - ldap_user.mail = u.email - processed_accounts[UPDATED].add(u.account) - users[UPDATED].add(u) - except Exception: - _log.warning("Cannot change email address to %s for %s in LDAP" % (u.email, u.account.vsc_id)) - users[ERROR].add(u) - - return users - - -def sync_altered_accounts(last, now, dry_run=True): - """ - Add new users to the LDAP and update altered users. This does not include usergroups. - - @type last: datetime - @type now: datetime - @return: tuple (new, updated, error) that indicates what accounts were new, changed or could not be altered. - """ - changed_accounts = Account.objects.filter(modify_timestamp__range=[last, now]) - - accounts = { - NEW: set(), - UPDATED: set(), - ERROR: set(), - } + def __init__(self): + self.processed_accounts = None + self.processed_pubkeys = None + self.client = AccountpageClient() # TODO: things here + pass + + def add_or_update(self, VscLdapKlass, cn, ldap_attributes, dry_run): + """ + Perform the update in LDAP for the given vsc.ldap.entitities class, cn and the ldap attributes. + + @return: NEW, UPDATED or ERROR, depending on the operation and its result. + """ + ldap_entries = self.vscldapclass.lookup(CnFilter(cn)) + if not ldap_entries: + # add the entry + _log.debug("add new entry %s %s with the attributes %s", VscLdapKlass.__name__, cn, ldap_attributes) + + if not dry_run: + try: + entry = VscLdapKlass(cn) + entry.add(ldap_attributes) + _log.info("Added a new user %s to LDAP" % (cn,)) + except LDAPError: + _log.warning("Could not add %s %s to LDAP" % (VscLdapKlass.__name__, cn,)) + return ERROR + return NEW + else: + ldap_entries[0].status + _log.debug("update existing entry %s %s with the attributes %s -- old entry: %s", + VscLdapKlass.__name__, cn, ldap_attributes, ldap_entries[0].ldap_info) + + if not dry_run: + try: + ldap_entries[0].modify_ldap(ldap_attributes) + _log.info("Modified %s %s in LDAP" % (VscLdapKlass.__name__, cn,)) + except LDAPError: + _log.warning("Could not add %s %s to LDAP" % (VscLdapKlass.__name__, cn,)) + return ERROR + return UPDATED + + + def get_public_keys(self, vsc_id): + """Get a list of public keys for a given vsc id""" + pks = [mkVscAccountPubkey(p) for p in self.client.account[p.vsc_id] if not p.deleted] + if not pks: + pks = [ACCOUNT_WITHOUT_PUBLIC_KEYS_MAGIC_STRING] + return pks + + def sync_altered_pubkeys(last, dry_run=True): + """ + Remove obsolete public keys from the LDAP and add new public keys to the LDAP. + """ + changed_pubkeys = [mkVscAccountPubkey(p) for p in self.client.account.pubkey.modified.now.get()] + + pubkeys = { + UPDATED: set(), + DONE: set(), + ERROR: set(), + } - sync_accounts = list(changed_accounts) + new_pubkeys = [p for p in changed_pubkeys if not p.deleted] + deleted_pubkeys = [p for p in changed_pubkeys if p.deleted] - _log.info("Found %d modified accounts in the range %s until %s" % (len(sync_accounts), - last.strftime("%Y%m%d%H%M%SZ"), - now.strftime("%Y%m%d%H%M%SZ"))) - _log.debug("Modified accounts: %s", [a.vsc_id for a in sync_accounts]) + _log.warning("Deleted pubkeys %s" % (deleted_pubkeys,)) + _log.debug("New pubkeys %s", new_pubkeys) + for p in changed_pubkeys: - for account in sync_accounts: + if not p.vsc_id: + # This should NOT happen + _log.error("Key %d had no associated user anymore",p) + continue - try: - home_storage = Storage.objects.get(storage_type=settings.HOME, institute=account.user.person.institute) - home_quota = UserSizeQuota.objects.get(user=account, storage=home_storage).hard - except UserSizeQuota.DoesNotExist: - home_quota = 0 - _log.warning("Could not find quota information for %s on %s, setting to 0" % (account.vsc_id, home_storage.name)) - except Storage.DoesNotExist: - home_quota = 0 - _log.warning("No home storage for institute %s defined, setting quota to 0" % (account.user.person.institute,)) - except User.DoesNotExist: - _log.error("No corresponding User for account %s" % (account.vsc_id,)) - continue - except Person.DoesNotExist: - _log.error("No corresponding Person for account %s" % (account.vsc_id,)) - continue + try: + account = mkVscAccount(self.client.account[p.vsc_id]) + except HTTPError: + _log.warning("No account for the user %s corresponding to the public key %d" % (p.vsc_id, p)) + continue - try: - data_storage = Storage.objects.get(storage_type=settings.DATA, institute=account.user.person.institute) - data_quota_ = UserSizeQuota.objects.filter(user=account, storage=data_storage) - if len(list(data_quota_)) > 1: - # this is the UGent case; we need to further distinguish between our various filesets, in - # this case the vscxyz fileset - data_quota = data_quota_.get(fileset=account.vsc_id[:6]).hard - else: - data_quota = data_quota_[0].hard - except (UserSizeQuota.DoesNotExist, IndexError): - data_quota = 0 - _log.warning("Could not find quota information for %s on %s, setting to 0" % (account.vsc_id, data_storage.name)) - except Storage.DoesNotExist: - data_quota = 0 - _log.warning("No data storage for institute %s defined, setting quota to 0" % (account.user.person.institute,)) + if account in self.processed_accounts[NEW] or account in self.processed_accounts[UPDATED]: + _log.info("Account %s was already processed and has the new set of public keys" % (account.vsc_id,)) + pubkeys[DONE].add(p) + continue - try: - scratch_storage = Storage.objects.filter(storage_type=settings.SCRATCH, institute=account.user.person.institute) - if not scratch_storage: - raise Storage.DoesNotExist("No scratch storage for institute %s" % (account.user.person.institute,)) + try: + ldap_user = VscLdapUser(p.vsc_id) + ldap_user.pubkey = [pk.pubkey for pk in pks] + self.processed_accounts[UPDATED].add(account) + pubkeys[UPDATED].add(p) + except Exception: + _log.warning("Cannot add pubkey for account %s to LDAP" % (account.vsc_id,)) + pubkeys[ERROR].add(p) + + self.processed_pubkeys = pubkeys + + + def sync_altered_accounts(self, last, dry_run=True): + """ + Add new users to the LDAP and update altered users. This does not include usergroups. + + @type last: datetime + @type now: datetime + @return: tuple (new, updated, error) that indicates what accounts were new, changed or could not be altered. + """ + changed_accounts= [mkVscAccount(a) for a in self.client.account.modified.now.get()] + now = datetime.now() + + accounts = { + NEW: set(), + UPDATED: set(), + ERROR: set(), + } - if account.user.person.institute in (Site.objects.get(site=GENT),): - scratch_storage = scratch_storage.filter(name='VSC_SCRATCH_DELCATTY')[0] # TODO: This can be ignored once we go to sync from django, skipping the LDAP completely - else: - scratch_storage = scratch_storage[0] + sync_accounts = list(changed_accounts) - scratch_quota_ = UserSizeQuota.objects.filter(user=account, storage=scratch_storage) # take the first one - if len(list(scratch_quota_)) > 1: - # this is the UGent case; we need to further distinguish between our various filesets, in - # this case the vscxyz fileset - scratch_quota = scratch_quota_.get(fileset=account.vsc_id[:6]).hard - else: - scratch_quota = scratch_quota_[0].hard - except (UserSizeQuota.DoesNotExist, IndexError): - scratch_quota = 0 - _log.warning("Could not find quota information for %s on %s, setting to 0" % (account.vsc_id, scratch_storage.name)) - except Storage.DoesNotExist: - scratch_quota = 0 - _log.warning("No scratch storage for institute %s defined, setting quota to 0" % (account.user.person.institute,)) + _log.info("Found %d modified accounts in the range %s until %s" % (len(sync_accounts), + last.strftime("%Y%m%d%H%M%SZ"), + now.strftime("%Y%m%d%H%M%SZ"))) + _log.debug("Modified accounts: %s", [a.vsc_id for a in sync_accounts]) - try: + for account in sync_accounts: + try: + #TODO: get usergroup here + pass + except UserGroup.DoesNotExist: + _log.error("No corresponding UserGroup for user %s" % (account.vsc_id,)) + continue try: gecos = str(account.user.person.gecos) except UnicodeEncodeError: - gecos = account.user.person.gecos.encode('ascii', 'ignore') + gecos = account.person.gecos.encode('ascii', 'ignore') _log.warning("Converting unicode to ascii for gecos resulting in %s", gecos) - public_keys = [str(p.pubkey) for p in Pubkey.objects.filter(user=account.user, deleted=False)] - if not public_keys: - public_keys = [ACCOUNT_WITHOUT_PUBLIC_KEYS_MAGIC_STRING] + public_keys = self.get_public_keys(account.vsc_id) ldap_attributes = { 'cn': str(account.vsc_id), 'uidNumber': ["%s" % (account.vsc_id_number,)], 'gecos': [gecos], - 'mail': [str(account.user.email)], - 'institute': [str(account.user.person.institute.site)], - 'instituteLogin': [str(account.user.person.institute_login)], + 'mail': [str(account.email)], + 'institute': [str(account.person.institute)], + 'instituteLogin': [str(account.person.institute_login)], 'uid': [str(account.vsc_id)], 'homeDirectory': [str(account.home_directory)], 'dataDirectory': [str(account.data_directory)], 'scratchDirectory': [str(account.scratch_directory)], - 'homeQuota': ["%d" % (home_quota,)], - 'dataQuota': ["%d" % (data_quota,)], - 'scratchQuota': ["%d" % (scratch_quota,)], + #TODO: no longer needed? + #'homeQuota': ["%d" % (home_quota,)], + #'dataQuota': ["%d" % (data_quota,)], + #'scratchQuota': ["%d" % (scratch_quota,)], 'pubkey': public_keys, + # TODO get usergroup with api 'gidNumber': ["%s" % (account.usergroup.vsc_id_number,)], 'loginShell': [str(account.login_shell)], # 'mukHomeOnScratch': ["FALSE"], # FIXME, see #37 - 'researchField': ["unknown"], + 'researchField': [account.research_field], 'status': [str(account.status)], } - except UserGroup.DoesNotExist: - _log.error("No corresponding UserGroup for user %s" % (account.vsc_id,)) - continue - result = add_or_update(VscLdapUser, account.vsc_id, ldap_attributes, dry_run) - accounts[result].add(account) + result = add_or_update(VscLdapUser, account.vsc_id, ldap_attributes, dry_run) + accounts[result].add(account) - return accounts + self.processed_accounts = accounts -def sync_altered_user_quota(last, now, altered_accounts, dry_run=True): - """ - Check for users who have altered quota and sync these to the LDAP. - - @type last: datetime - @type now: datetime - @return: tuple (new, updated, error) that indicates what accounts were new, changed or could not be altered. - """ + def sync_altered_user_groups(last, now, dry_run=True): + """ + Add new usergroups to the LDAP and update altered usergroups. - changed_quota = UserSizeQuota.objects.filter(modify_timestamp__range=[last, now]) + @type last: datetime + @type now: datetime + @return: tuple (new, updated, error) that indicates what usergroups were new, changed or could not be altered. + """ - _log.info("Found %d modified quota in the range %s until %s" % (len(changed_quota), - last.strftime("%Y%m%d%H%M%SZ"), - now.strftime("%Y%m%d%H%M%SZ"))) - quotas = { - NEW: set(), - UPDATED: set(), - ERROR: set(), - DONE: set(), - } + changed_usergroups = UserGroup.objects.filter(modify_timestamp__range=[last, now]) - for quota in changed_quota: - account = quota.user - if account in altered_accounts[NEW] or account in altered_accounts[UPDATED]: - _log.info("Quota %s was already processed with account %s" % (quota, account.vsc_id)) - quotas[DONE].add(quota) - continue + _log.info("Found %d modified usergroups in the range %s until %s" % (len(changed_usergroups), + last.strftime("%Y%m%d%H%M%SZ"), + now.strftime("%Y%m%d%H%M%SZ"))) - try: - ldap_user = VscLdapUser(account.vsc_id) - ldap_user.status - if quota.storage.storage_type in (settings.HOME,): - ldap_user.homeQuota = "%d" % (quota.hard,) - quotas[UPDATED].add(quota) - elif quota.storage.storage_type in (settings.DATA,): - ldap_user.dataQuota = "%d" % (quota.hard,) - quotas[UPDATED].add(quota) - elif quota.storage.storage_type in (settings.SCRATCH,): - ldap_user.scratchQuota = "%d" % (quota.hard,) - quotas[UPDATED].add(quota) - else: - _log.warning("Cannot sync quota to LDAP (storage type %s unknown)" % (quota.storage.storage_type,)) + _log.debug("Modified usergroups: %s", [g.vsc_id for g in changed_usergroups]) - except Exception: - _log.warning("Cannot update quota %s" % (quota,)) - quotas[ERROR].add(quota) + groups = { + NEW: set(), + UPDATED: set(), + ERROR: set(), + } - return quotas + for usergroup in changed_usergroups: -def sync_altered_user_groups(last, now, dry_run=True): - """ - Add new usergroups to the LDAP and update altered usergroups. + ldap_attributes = { + 'cn': str(usergroup.vsc_id), + 'institute': [str(usergroup.institute.site)], + 'gidNumber': ["%d" % (usergroup.vsc_id_number,)], + 'moderator': [str(usergroup.vsc_id)], # a fixed single moderator! + 'memberUid': [str(usergroup.vsc_id)], # a single member + 'status': [str(usergroup.status)], + } - @type last: datetime - @type now: datetime - @return: tuple (new, updated, error) that indicates what usergroups were new, changed or could not be altered. - """ + result = add_or_update(VscLdapGroup, usergroup.vsc_id, ldap_attributes, dry_run) + groups[result].add(usergroup) - changed_usergroups = UserGroup.objects.filter(modify_timestamp__range=[last, now]) + return groups - _log.info("Found %d modified usergroups in the range %s until %s" % (len(changed_usergroups), - last.strftime("%Y%m%d%H%M%SZ"), - now.strftime("%Y%m%d%H%M%SZ"))) - _log.debug("Modified usergroups: %s", [g.vsc_id for g in changed_usergroups]) + def sync_altered_autogroups(dry_run=True): - groups = { - NEW: set(), - UPDATED: set(), - ERROR: set(), - } + changed_autogroups = Autogroup.objects.all() # we always sync autogroups since we cannot know beforehand if their membership list changed - for usergroup in changed_usergroups: + _log.info("Found %d autogroups" % (len(changed_autogroups),)) + _log.debug("Autogroups: %s", [a.vsc_id for a in changed_autogroups]) - ldap_attributes = { - 'cn': str(usergroup.vsc_id), - 'institute': [str(usergroup.institute.site)], - 'gidNumber': ["%d" % (usergroup.vsc_id_number,)], - 'moderator': [str(usergroup.vsc_id)], # a fixed single moderator! - 'memberUid': [str(usergroup.vsc_id)], # a single member - 'status': [str(usergroup.status)], + groups = { + NEW: set(), + UPDATED: set(), + ERROR: set(), } - result = add_or_update(VscLdapGroup, usergroup.vsc_id, ldap_attributes, dry_run) - groups[result].add(usergroup) + for autogroup in changed_autogroups: - return groups + ldap_attributes = { + 'cn': str(autogroup.vsc_id), + 'institute': [str(autogroup.institute.site)], + 'gidNumber': ["%d" % (autogroup.vsc_id_number,)], + 'moderator': [str(u.account.vsc_id) for u in DGroup.objects.get(name='administrator_%s' % (autogroup.institute.site,)).user_set.all()], + 'memberUid': [str(a.vsc_id) for a in autogroup.get_members()], + 'status': [str(autogroup.status)], + 'autogroup': [str(s.vsc_id) for s in autogroup.sources.all()], + } + _log.debug("Proposed changes for autogroup %s: %s", autogroup.vsc_id, ldap_attributes) -def sync_altered_autogroups(dry_run=True): + result = add_or_update(VscLdapGroup, autogroup.vsc_id, ldap_attributes, dry_run) + groups[result].add(autogroup) - changed_autogroups = Autogroup.objects.all() # we always sync autogroups since we cannot know beforehand if their membership list changed + return groups - _log.info("Found %d autogroups" % (len(changed_autogroups),)) - _log.debug("Autogroups: %s", [a.vsc_id for a in changed_autogroups]) - groups = { - NEW: set(), - UPDATED: set(), - ERROR: set(), - } + def sync_altered_group_membership(last, now, processed_groups, dry_run=True): + """ + Synchronise the memberships for groups when users are added/removed. + """ + changed_members = Membership.objects.filter(modify_timestamp__range=[last, now]) - for autogroup in changed_autogroups: + _log.info("Found %d modified members in the range %s until %s" % (len(changed_members), + last.strftime("%Y%m%d%H%M%SZ"), + now.strftime("%Y%m%d%H%M%SZ"))) + _log.debug("Modified members: %s", [m.account.vsc_id for m in changed_members]) - ldap_attributes = { - 'cn': str(autogroup.vsc_id), - 'institute': [str(autogroup.institute.site)], - 'gidNumber': ["%d" % (autogroup.vsc_id_number,)], - 'moderator': [str(u.account.vsc_id) for u in DGroup.objects.get(name='administrator_%s' % (autogroup.institute.site,)).user_set.all()], - 'memberUid': [str(a.vsc_id) for a in autogroup.get_members()], - 'status': [str(autogroup.status)], - 'autogroup': [str(s.vsc_id) for s in autogroup.sources.all()], + members = { + NEW: set(), + UPDATED: set(), + ERROR: set(), + DONE: set(), } - _log.debug("Proposed changes for autogroup %s: %s", autogroup.vsc_id, ldap_attributes) + newly_processed_groups = set() + for member in changed_members: - result = add_or_update(VscLdapGroup, autogroup.vsc_id, ldap_attributes, dry_run) - groups[result].add(autogroup) + if member.group in processed_groups[NEW] \ + or member.group in processed_groups[UPDATED] \ + or member.group in newly_processed_groups: + _log.info("Member %s was already processed with group %s" % (member.account.vsc_id, member.group)) + members[DONE].add(member) + continue - return groups + try: + ldap_group = VscLdapGroup(member.group.vsc_id) + ldap_group.status + ldap_group.memberUid = [str(m.account.vsc_id) for m in Membership.objects.filter(group=member.group)] + except Exception: + _log.warning("Cannot add member %s to group %s" % (member.account.vsc_id, member.group.vsc_id)) + members[ERROR].add(member) + else: + newly_processed_groups.add(member.group) + members[UPDATED].add(member) + _log.info("Processed group %s member %s" % (member.group.vsc_id, member.account.vsc_id)) + return members -def sync_altered_group_membership(last, now, processed_groups, dry_run=True): - """ - Synchronise the memberships for groups when users are added/removed. - """ - changed_members = Membership.objects.filter(modify_timestamp__range=[last, now]) - _log.info("Found %d modified members in the range %s until %s" % (len(changed_members), - last.strftime("%Y%m%d%H%M%SZ"), - now.strftime("%Y%m%d%H%M%SZ"))) - _log.debug("Modified members: %s", [m.account.vsc_id for m in changed_members]) + def sync_altered_groups(last, now, dry_run=True): + """ + Synchronise altered groups back to LDAP. + """ + changed_groups = Group.objects.filter(modify_timestamp__range=[last, now]) - members = { - NEW: set(), - UPDATED: set(), - ERROR: set(), - DONE: set(), - } + _log.info("Found %d modified groups in the range %s until %s" % (len(changed_groups), + last.strftime("%Y%m%d%H%M%SZ"), + now.strftime("%Y%m%d%H%M%SZ"))) + _log.debug("Modified groups: %s", [g.vsc_id for g in changed_groups]) + groups = { + NEW: set(), + UPDATED: set(), + ERROR: set(), + } + for group in changed_groups: - newly_processed_groups = set() - for member in changed_members: + ldap_attributes = { + 'cn': str(group.vsc_id), + 'institute': [str(group.institute.site)], + 'gidNumber': ["%d" % (group.vsc_id_number,)], + 'moderator': [str(m.account.vsc_id) for m in Membership.objects.filter(moderator=True, group=group)], + 'memberUid': [str(a.vsc_id) for a in group.get_members()], + 'status': [str(group.status)], + } - if member.group in processed_groups[NEW] \ - or member.group in processed_groups[UPDATED] \ - or member.group in newly_processed_groups: - _log.info("Member %s was already processed with group %s" % (member.account.vsc_id, member.group)) - members[DONE].add(member) - continue + _log.debug("Proposed changes for group %s: %s", group.vsc_id, ldap_attributes) - try: - ldap_group = VscLdapGroup(member.group.vsc_id) - ldap_group.status - ldap_group.memberUid = [str(m.account.vsc_id) for m in Membership.objects.filter(group=member.group)] - except Exception: - _log.warning("Cannot add member %s to group %s" % (member.account.vsc_id, member.group.vsc_id)) - members[ERROR].add(member) - else: - newly_processed_groups.add(member.group) - members[UPDATED].add(member) - _log.info("Processed group %s member %s" % (member.group.vsc_id, member.account.vsc_id)) + result = add_or_update(VscLdapGroup, group.vsc_id, ldap_attributes, dry_run) + groups[result].add(group) - return members + return groups -def sync_altered_groups(last, now, dry_run=True): - """ - Synchronise altered groups back to LDAP. - """ - changed_groups = Group.objects.filter(modify_timestamp__range=[last, now]) - - _log.info("Found %d modified groups in the range %s until %s" % (len(changed_groups), - last.strftime("%Y%m%d%H%M%SZ"), - now.strftime("%Y%m%d%H%M%SZ"))) - _log.debug("Modified groups: %s", [g.vsc_id for g in changed_groups]) - groups = { - NEW: set(), - UPDATED: set(), - ERROR: set(), - } - for group in changed_groups: - - ldap_attributes = { - 'cn': str(group.vsc_id), - 'institute': [str(group.institute.site)], - 'gidNumber': ["%d" % (group.vsc_id_number,)], - 'moderator': [str(m.account.vsc_id) for m in Membership.objects.filter(moderator=True, group=group)], - 'memberUid': [str(a.vsc_id) for a in group.get_members()], - 'status': [str(group.status)], + def sync_altered_vo_membership(last, now, processed_vos, dry_run=True): + """ + Synchronise the memberships for groups when users are added/removed. + """ + changed_members = VoMembership.objects.filter(modify_timestamp__range=[last, now]) + + _log.info("Found %d modified members in the range %s until %s" % (len(changed_members), + last.strftime("%Y%m%d%H%M%SZ"), + now.strftime("%Y%m%d%H%M%SZ"))) + _log.debug("Modified VO members: %s", [m.account.vsc_id for m in changed_members]) + members = { + NEW: set(), + UPDATED: set(), + ERROR: set(), + DONE: set(), } + newly_processed_vos = set() - _log.debug("Proposed changes for group %s: %s", group.vsc_id, ldap_attributes) + for member in changed_members: - result = add_or_update(VscLdapGroup, group.vsc_id, ldap_attributes, dry_run) - groups[result].add(group) + if member.group in processed_vos[NEW] \ + or member.group in processed_vos[UPDATED] \ + or member.group in newly_processed_vos: + _log.info("Member %s membership was already processed with group %s" % (member.account.vsc_id, member.group)) + members[DONE].add(member) + continue - return groups + try: + ldap_group = VscLdapGroup(member.group.vsc_id) + ldap_group.status + ldap_group.memberUid = [str(m.account.vsc_id) for m in VoMembership.objects.filter(group=member.group)] + except Exception: + _log.warning("Cannot add member %s to group %s" % (member.account.vsc_id, member.group.vsc_id)) + members[ERROR].add(member) + else: + newly_processed_vos.add(member.group) + members[UPDATED].add(member) + _log.info("Processed group %s member %s" % (member.group.vsc_id, member.account.vsc_id)) + return members -def sync_altered_vo_membership(last, now, processed_vos, dry_run=True): - """ - Synchronise the memberships for groups when users are added/removed. - """ - changed_members = VoMembership.objects.filter(modify_timestamp__range=[last, now]) - _log.info("Found %d modified members in the range %s until %s" % (len(changed_members), + def sync_altered_VO(last, now, dry_run=True): + """ + Synchronise altered VOs back to the LDAP. + """ + changed_vos = VirtualOrganisation.objects.filter(modify_timestamp__range=[last, now]) + _log.info("Found %d modified vos in the range %s until %s" % (len(changed_vos), last.strftime("%Y%m%d%H%M%SZ"), now.strftime("%Y%m%d%H%M%SZ"))) - _log.debug("Modified VO members: %s", [m.account.vsc_id for m in changed_members]) - members = { - NEW: set(), - UPDATED: set(), - ERROR: set(), - DONE: set(), - } - newly_processed_vos = set() - - for member in changed_members: - - if member.group in processed_vos[NEW] \ - or member.group in processed_vos[UPDATED] \ - or member.group in newly_processed_vos: - _log.info("Member %s membership was already processed with group %s" % (member.account.vsc_id, member.group)) - members[DONE].add(member) - continue - - try: - ldap_group = VscLdapGroup(member.group.vsc_id) - ldap_group.status - ldap_group.memberUid = [str(m.account.vsc_id) for m in VoMembership.objects.filter(group=member.group)] - except Exception: - _log.warning("Cannot add member %s to group %s" % (member.account.vsc_id, member.group.vsc_id)) - members[ERROR].add(member) - else: - newly_processed_vos.add(member.group) - members[UPDATED].add(member) - _log.info("Processed group %s member %s" % (member.group.vsc_id, member.account.vsc_id)) - - return members - - -def sync_altered_VO(last, now, dry_run=True): - """ - Synchronise altered VOs back to the LDAP. - """ - changed_vos = VirtualOrganisation.objects.filter(modify_timestamp__range=[last, now]) - _log.info("Found %d modified vos in the range %s until %s" % (len(changed_vos), - last.strftime("%Y%m%d%H%M%SZ"), - now.strftime("%Y%m%d%H%M%SZ"))) - _log.debug("Modified VOs: %s", [v.vsc_id for v in changed_vos]) - - vos = { - NEW: set(), - UPDATED: set(), - ERROR: set(), - } - - for vo in changed_vos: - - try: - data_storage = Storage.objects.get(storage_type=settings.DATA, institute=vo.institute) - data_quota = VirtualOrganisationSizeQuota.objects.get(virtual_organisation=vo, storage=data_storage, fileset=vo.vsc_id).hard - except (VirtualOrganisationSizeQuota.DoesNotExist, IndexError): - data_quota = 0 - _log.warning("Could not find VO quota information for %s on %s, setting to 0" % (vo.vsc_id, data_storage.name)) - except Storage.DoesNotExist: - data_quota = 0 - _log.warning("No VO data storage for institute %s defined, setting quota to 0" % (vo.institute,)) + _log.debug("Modified VOs: %s", [v.vsc_id for v in changed_vos]) - try: - scratch_storage = Storage.objects.filter(storage_type=settings.SCRATCH, institute=vo.institute) - scratch_quota = VirtualOrganisationSizeQuota.objects.get(virtual_organisation=vo, storage=scratch_storage[0], fileset=vo.vsc_id).hard # take the first one - except (VirtualOrganisationSizeQuota.DoesNotExist, IndexError): - scratch_quota = 0 - _log.warning("Could not find VO quota information for %s on %s, setting to 0" % (vo.vsc_id, data_storage.name)) - except Storage.DoesNotExist: - scratch_quota = 0 - _log.warning("No VO scratch storage for institute %s defined, setting quota to 0" % (vo.institute,)) - - # Hack to deal with the anomaly that the VO admin actually 'belongs' to multiple VOs, only in the LDAP - # the moderator need not be a member - if vo.vsc_id in settings.VSC.institute_vos.values(): - moderators = ['vsc40024'] - else: - moderators = [str(m.account.vsc_id) for m in VoMembership.objects.filter(moderator=True, group=vo)] - - ldap_attributes = { - 'cn': str(vo.vsc_id), - 'institute': [str(vo.institute.site)], - 'gidNumber': ["%d" % (vo.vsc_id_number,)], - 'moderator': moderators, - 'memberUid': [str(m.account.vsc_id) for m in VoMembership.objects.filter(group=vo)], - 'status': [str(vo.status)], - 'fairshare': ["%d" % (vo.fairshare,)], - 'description': [str(vo.description)], - 'dataDirectory': [str(vo.data_path)], - 'scratchDirectory': [str(vo.scratch_path)], - 'dataQuota': [str(data_quota)], - 'scratchQuota': [str(scratch_quota)], + vos = { + NEW: set(), + UPDATED: set(), + ERROR: set(), } - _log.debug("Proposed changes for VO %s: %s", vo.vsc_id, ldap_attributes) - - result = add_or_update(VscLdapGroup, vo.vsc_id, ldap_attributes, dry_run) - vos[result].add(vo) - - return vos + for vo in changed_vos: + # Hack to deal with the anomaly that the VO admin actually 'belongs' to multiple VOs, only in the LDAP + # the moderator need not be a member + if vo.vsc_id in settings.VSC.institute_vos.values(): + moderators = ['vsc40024'] + else: + moderators = [str(m.account.vsc_id) for m in VoMembership.objects.filter(moderator=True, group=vo)] -def sync_altered_vo_quota(last, now, altered_vos, dry_run=True): - """ - Sync the changed quota for the VO to the LDAP - """ - changed_quota = VirtualOrganisationSizeQuota.objects.filter(virtual_organisation__status=ACTIVE, modify_timestamp__range=[last, now]) - - _log.info("Found %d modified VO quota in the range %s until %s" % (len(changed_quota), - last.strftime("%Y%m%d%H%M%SZ"), - now.strftime("%Y%m%d%H%M%SZ"))) - quotas = { - NEW: set(), - UPDATED: set(), - ERROR: set(), - DONE: set(), - } + ldap_attributes = { + 'cn': str(vo.vsc_id), + 'institute': [str(vo.institute.site)], + 'gidNumber': ["%d" % (vo.vsc_id_number,)], + 'moderator': moderators, + 'memberUid': [str(m.account.vsc_id) for m in VoMembership.objects.filter(group=vo)], + 'status': [str(vo.status)], + 'fairshare': ["%d" % (vo.fairshare,)], + 'description': [str(vo.description)], + 'dataDirectory': [str(vo.data_path)], + 'scratchDirectory': [str(vo.scratch_path)], + #TODO: what do here? + #'dataQuota': [str(data_quota)], + #'scratchQuota': [str(scratch_quota)], + } - for quota in changed_quota: - virtual_organisation = quota.virtual_organisation - if virtual_organisation in altered_vos[NEW] or virtual_organisation in altered_vos[UPDATED]: - _log.info("Quota %s was already processed with VO %s" % (quota, virtual_organisation.vsc_id)) - quotas[DONE].add(quota) - continue + _log.debug("Proposed changes for VO %s: %s", vo.vsc_id, ldap_attributes) - try: - ldap_group = VscLdapGroup(virtual_organisation.vsc_id) - ldap_group.status - if quota.storage.storage_type in (settings.HOME,): - ldap_group.homeQuota = "%d" % (quota.hard,) - quotas[UPDATED].add(quota) - elif quota.storage.storage_type in (settings.DATA,): - ldap_group.dataQuota = "%d" % (quota.hard,) - quotas[UPDATED].add(quota) - elif quota.storage.storage_type in (settings.SCRATCH,): - ldap_group.scratchQuota = "%d" % (quota.hard,) - quotas[UPDATED].add(quota) - else: - _log.warning("Cannot sync quota to LDAP (storage type %s unknown)" % (quota.storage.storage_type,)) + result = add_or_update(VscLdapGroup, vo.vsc_id, ldap_attributes, dry_run) + vos[result].add(vo) - except Exception: - _log.warning("Cannot update quota %s" % (quota,)) - quotas[ERROR].add(quota) + return vos - return quotas def main(): - now = datetime.utcnow().replace(tzinfo=utc) + now = datetime.utcnow().replace(tzinfo=timezone.utc) options = { 'nagios-check-interval-threshold': NAGIOS_CHECK_INTERVAL_THRESHOLD, @@ -673,7 +475,8 @@ def main(): opts = ExtendedSimpleOption(options) stats = {} - l = LdapQuery(VscConfiguration(VSC_CONF_DEFAULT_FILENAME)) + # Creating this here because this is a singleton class + _ = LdapQuery(VscConfiguration(VSC_CONF_DEFAULT_FILENAME)) last_timestamp = opts.options.start_timestamp if not last_timestamp: @@ -713,14 +516,14 @@ def main(): except OSError: _log.raiseException("Could not drop privileges") - last = datetime.strptime(last_timestamp, "%Y%m%d%H%M%SZ").replace(tzinfo=utc) - - altered_accounts = sync_altered_accounts(last, now, opts.options.dry_run) - altered_pubkeys = sync_altered_pubkeys(last, now, altered_accounts, opts.options.dry_run) + last = datetime.strptime(last_timestamp, "%Y%m%d%H%M%SZ").replace(tzinfo=timezone.utc) + syncer = LdapSyncer() + syncer.sync_altered_accounts(last, now, opts.options.dry_run) + syncer.sync_altered_pubkeys(last, now, opts.options.dry_run) # altered_users = sync_altered_users(last, now, altered_accounts) # FIXME: no modification timestamps here :( - _log.debug("Altered accounts: %s" % (altered_accounts,)) - _log.debug("Altered pubkeys: %s" % (altered_pubkeys,)) + _log.debug("Altered accounts: %s", syncer.processed_accounts) + _log.debug("Altered pubkeys: %s", syncer.altered_pubkeys) # _log.debug("Altered users: %s" % (altered_users,)) altered_usergroups = sync_altered_user_groups(last, now, opts.options.dry_run) @@ -731,8 +534,6 @@ def main(): altered_vos = sync_altered_VO(last, now, opts.options.dry_run) altered_vo_members = sync_altered_vo_membership(last, now, altered_vos, opts.options.dry_run) - altered_user_quota = sync_altered_user_quota(last, now, altered_accounts, opts.options.dry_run) - altered_vo_quota = sync_altered_vo_quota(last, now, altered_vos, opts.options.dry_run) _log.debug("Altered usergroups: %s" % (altered_usergroups,)) _log.debug("Altered autogroups: %s" % (altered_autogroups,)) @@ -740,8 +541,6 @@ def main(): _log.debug("Altered members: %s" % (altered_members,)) _log.debug("Altered VOs: %s" % (altered_vos,)) _log.debug("Altered VO members: %s" % (altered_vo_members,)) - _log.debug("Altered user quota: %s" % (altered_user_quota,)) - _log.debug("Altered VO quota: %s" % (altered_vo_quota,)) if not altered_accounts[ERROR] \ and not altered_groups[ERROR] \ @@ -749,8 +548,6 @@ def main(): and not altered_members[ERROR] \ and not altered_vo_members[ERROR] \ and not altered_usergroups[ERROR] \ - and not altered_user_quota[ERROR] \ - and not altered_vo_quota[ERROR]: _log.info("Child process exiting correctly") sys.exit(0) else: @@ -763,8 +560,6 @@ def main(): ("altered members", altered_members[ERROR]), ("altered vo_members", altered_vo_members[ERROR]), ("altered usergroups", altered_usergroups[ERROR]), - ("altered user quota", altered_user_quota[ERROR]), - ("altered vo quota", altered_vo_quota[ERROR]), ]] )) sys.exit(-1) diff --git a/lib/vsc/administration/user.py b/lib/vsc/administration/user.py index b4a2c0cf..1062062f 100644 --- a/lib/vsc/administration/user.py +++ b/lib/vsc/administration/user.py @@ -32,9 +32,9 @@ from urllib2 import HTTPError from vsc.utils import fancylogger -from vsc.accountpage.wrappers import VscAccount, VscAccountPerson, VscAccountPubkey, VscHomeOnScratch, VscUserGroup +from vsc.accountpage.wrappers import mkVscAccountPerson, mkVscAccountPubkey, mkVscHomeOnScratch, mkVscUserGroup from vsc.accountpage.wrappers import mkVscAccount, mkUserGroup -from vsc.accountpage.wrappers import VscGroup, VscUserSizeQuota +from vsc.accountpage.wrappers import mkVscGroup, mkVscUserSizeQuota from vsc.administration.tools import create_stat_directory from vsc.config.base import VSC, Muk, VscStorage, VSC_DATA, VSC_HOME from vsc.config.base import NEW, MODIFIED, MODIFY, ACTIVE @@ -64,17 +64,17 @@ def __init__(self, user_id, rest_client): # We immediately retrieve this information try: - self.account = VscAccount(**(rest_client.account[user_id].get()[1])) - self.person = VscAccountPerson(**(rest_client.account[user_id].person.get()[1])) - self.pubkeys = [VscAccountPubkey(**p) for p in rest_client.account[user_id].pubkey.get()[1] + self.account = mkVscAccount((rest_client.account[user_id].get()[1])) + self.person = mkVscAccountPerson((rest_client.account[user_id].person.get()[1])) + self.pubkeys = [mkVscAccountPubkey(p) for p in rest_client.account[user_id].pubkey.get()[1] if not p['deleted']] if self.person.institute_login in ('x_admin', 'admin', 'voadmin'): # TODO to be removed when magic site admin usergroups are pruged from code - self.usergroup = VscGroup(**(rest_client.group[user_id].get())[1]) + self.usergroup = mkVscGroup((rest_client.group[user_id].get())[1]) else: - self.usergroup = VscUserGroup(**(rest_client.account[user_id].usergroup.get()[1])) + self.usergroup = mkVscUserGroup((rest_client.account[user_id].usergroup.get()[1])) self.home_on_scratch = [ - VscHomeOnScratch(**h) for h in rest_client.account[user_id].home_on_scratch.get()[1] + mkVscHomeOnScratch(h) for h in rest_client.account[user_id].home_on_scratch.get()[1] ] except HTTPError: logging.error("Cannot get information from the account page") @@ -113,7 +113,7 @@ def __init__(self, user_id, storage=None, pickle_storage='VSC_SCRATCH_DELCATTY', def _init_quota(self, rest_client): try: - all_quota = [VscUserSizeQuota(**q) for q in rest_client.account[self.user_id].quota.get()[1]] + all_quota = [mkVscUserSizeQuota(q) for q in rest_client.account[self.user_id].quota.get()[1]] except HTTPError: logging.exception("Unable to retrieve quota information. Falling back to static info for home and data") self.user_home_quota = self.storage[VSC_HOME].user_quota From 55a69ee1db71f7393077b96f51ea1bf80fac861a Mon Sep 17 00:00:00 2001 From: Jens Timmerman Date: Tue, 21 Jun 2016 19:20:02 +0200 Subject: [PATCH 03/26] further progess of wip --- bin/sync_django_ldap.py | 251 +++++----------------------------------- 1 file changed, 32 insertions(+), 219 deletions(-) diff --git a/bin/sync_django_ldap.py b/bin/sync_django_ldap.py index 2627f577..b977b495 100644 --- a/bin/sync_django_ldap.py +++ b/bin/sync_django_ldap.py @@ -60,10 +60,7 @@ def class LdapSyncer(object): to the vsc ldap """ def __init__(self): - self.processed_accounts = None - self.processed_pubkeys = None self.client = AccountpageClient() # TODO: things here - pass def add_or_update(self, VscLdapKlass, cn, ldap_attributes, dry_run): """ @@ -111,7 +108,7 @@ def sync_altered_pubkeys(last, dry_run=True): """ Remove obsolete public keys from the LDAP and add new public keys to the LDAP. """ - changed_pubkeys = [mkVscAccountPubkey(p) for p in self.client.account.pubkey.modified.now.get()] + changed_pubkeys = [mkVscAccountPubkey(p) for p in self.client.account.pubkey.modified[last].get()[1]] pubkeys = { UPDATED: set(), @@ -160,11 +157,10 @@ def sync_altered_accounts(self, last, dry_run=True): Add new users to the LDAP and update altered users. This does not include usergroups. @type last: datetime - @type now: datetime @return: tuple (new, updated, error) that indicates what accounts were new, changed or could not be altered. """ - changed_accounts= [mkVscAccount(a) for a in self.client.account.modified.now.get()] - now = datetime.now() + changed_accounts= [mkVscAccount(a) for a in self.client.account.modified[last].get()[1]] + now = datetime.now() accounts = { NEW: set(), @@ -181,9 +177,8 @@ def sync_altered_accounts(self, last, dry_run=True): for account in sync_accounts: try: - #TODO: get usergroup here - pass - except UserGroup.DoesNotExist: + usergroup = mkUserGroup(client.account[account.vsc_id].usergroup.get()[1]) + except HTTPError: _log.error("No corresponding UserGroup for user %s" % (account.vsc_id,)) continue try: @@ -205,13 +200,12 @@ def sync_altered_accounts(self, last, dry_run=True): 'homeDirectory': [str(account.home_directory)], 'dataDirectory': [str(account.data_directory)], 'scratchDirectory': [str(account.scratch_directory)], - #TODO: no longer needed? + #TODO: fill in #'homeQuota': ["%d" % (home_quota,)], #'dataQuota': ["%d" % (data_quota,)], #'scratchQuota': ["%d" % (scratch_quota,)], 'pubkey': public_keys, - # TODO get usergroup with api - 'gidNumber': ["%s" % (account.usergroup.vsc_id_number,)], + 'gidNumber': [str(usergroup.vsc_id_number)], 'loginShell': [str(account.login_shell)], # 'mukHomeOnScratch': ["FALSE"], # FIXME, see #37 'researchField': [account.research_field], @@ -221,130 +215,15 @@ def sync_altered_accounts(self, last, dry_run=True): result = add_or_update(VscLdapUser, account.vsc_id, ldap_attributes, dry_run) accounts[result].add(account) - self.processed_accounts = accounts - - - def sync_altered_user_groups(last, now, dry_run=True): - """ - Add new usergroups to the LDAP and update altered usergroups. - - @type last: datetime - @type now: datetime - @return: tuple (new, updated, error) that indicates what usergroups were new, changed or could not be altered. - """ - - changed_usergroups = UserGroup.objects.filter(modify_timestamp__range=[last, now]) - - _log.info("Found %d modified usergroups in the range %s until %s" % (len(changed_usergroups), - last.strftime("%Y%m%d%H%M%SZ"), - now.strftime("%Y%m%d%H%M%SZ"))) - - _log.debug("Modified usergroups: %s", [g.vsc_id for g in changed_usergroups]) - - groups = { - NEW: set(), - UPDATED: set(), - ERROR: set(), - } - - for usergroup in changed_usergroups: - - ldap_attributes = { - 'cn': str(usergroup.vsc_id), - 'institute': [str(usergroup.institute.site)], - 'gidNumber': ["%d" % (usergroup.vsc_id_number,)], - 'moderator': [str(usergroup.vsc_id)], # a fixed single moderator! - 'memberUid': [str(usergroup.vsc_id)], # a single member - 'status': [str(usergroup.status)], - } - - result = add_or_update(VscLdapGroup, usergroup.vsc_id, ldap_attributes, dry_run) - groups[result].add(usergroup) - - return groups - - - def sync_altered_autogroups(dry_run=True): - - changed_autogroups = Autogroup.objects.all() # we always sync autogroups since we cannot know beforehand if their membership list changed - - _log.info("Found %d autogroups" % (len(changed_autogroups),)) - _log.debug("Autogroups: %s", [a.vsc_id for a in changed_autogroups]) - - groups = { - NEW: set(), - UPDATED: set(), - ERROR: set(), - } - - for autogroup in changed_autogroups: - - ldap_attributes = { - 'cn': str(autogroup.vsc_id), - 'institute': [str(autogroup.institute.site)], - 'gidNumber': ["%d" % (autogroup.vsc_id_number,)], - 'moderator': [str(u.account.vsc_id) for u in DGroup.objects.get(name='administrator_%s' % (autogroup.institute.site,)).user_set.all()], - 'memberUid': [str(a.vsc_id) for a in autogroup.get_members()], - 'status': [str(autogroup.status)], - 'autogroup': [str(s.vsc_id) for s in autogroup.sources.all()], - } - - _log.debug("Proposed changes for autogroup %s: %s", autogroup.vsc_id, ldap_attributes) - - result = add_or_update(VscLdapGroup, autogroup.vsc_id, ldap_attributes, dry_run) - groups[result].add(autogroup) - - return groups - - - def sync_altered_group_membership(last, now, processed_groups, dry_run=True): - """ - Synchronise the memberships for groups when users are added/removed. - """ - changed_members = Membership.objects.filter(modify_timestamp__range=[last, now]) - - _log.info("Found %d modified members in the range %s until %s" % (len(changed_members), - last.strftime("%Y%m%d%H%M%SZ"), - now.strftime("%Y%m%d%H%M%SZ"))) - _log.debug("Modified members: %s", [m.account.vsc_id for m in changed_members]) - - members = { - NEW: set(), - UPDATED: set(), - ERROR: set(), - DONE: set(), - } - - newly_processed_groups = set() - for member in changed_members: - - if member.group in processed_groups[NEW] \ - or member.group in processed_groups[UPDATED] \ - or member.group in newly_processed_groups: - _log.info("Member %s was already processed with group %s" % (member.account.vsc_id, member.group)) - members[DONE].add(member) - continue - - try: - ldap_group = VscLdapGroup(member.group.vsc_id) - ldap_group.status - ldap_group.memberUid = [str(m.account.vsc_id) for m in Membership.objects.filter(group=member.group)] - except Exception: - _log.warning("Cannot add member %s to group %s" % (member.account.vsc_id, member.group.vsc_id)) - members[ERROR].add(member) - else: - newly_processed_groups.add(member.group) - members[UPDATED].add(member) - _log.info("Processed group %s member %s" % (member.group.vsc_id, member.account.vsc_id)) - - return members + return accounts def sync_altered_groups(last, now, dry_run=True): """ Synchronise altered groups back to LDAP. + This also includes usergroups """ - changed_groups = Group.objects.filter(modify_timestamp__range=[last, now]) + changed_groups= [mkGroup(a) for a in self.client.allgroups.modified[last].get()[1]] _log.info("Found %d modified groups in the range %s until %s" % (len(changed_groups), last.strftime("%Y%m%d%H%M%SZ"), @@ -355,93 +234,29 @@ def sync_altered_groups(last, now, dry_run=True): UPDATED: set(), ERROR: set(), } + for group in changed_groups: + #TDOO: if is VO or autogroup, set autogroup sources or fairshare and scratch/data dirs + autogroup = False + vo = False + + ldap_attributes = { 'cn': str(group.vsc_id), - 'institute': [str(group.institute.site)], + 'institute': [str(group.institute)], 'gidNumber': ["%d" % (group.vsc_id_number,)], - 'moderator': [str(m.account.vsc_id) for m in Membership.objects.filter(moderator=True, group=group)], - 'memberUid': [str(a.vsc_id) for a in group.get_members()], + 'moderator': [str(m['vsc_id']) for m in group.moderators], + 'memberUid': [str(a['vsc_id') for a in group.members], 'status': [str(group.status)], } - _log.debug("Proposed changes for group %s: %s", group.vsc_id, ldap_attributes) - - result = add_or_update(VscLdapGroup, group.vsc_id, ldap_attributes, dry_run) - groups[result].add(group) - - return groups - - - def sync_altered_vo_membership(last, now, processed_vos, dry_run=True): - """ - Synchronise the memberships for groups when users are added/removed. - """ - changed_members = VoMembership.objects.filter(modify_timestamp__range=[last, now]) - - _log.info("Found %d modified members in the range %s until %s" % (len(changed_members), - last.strftime("%Y%m%d%H%M%SZ"), - now.strftime("%Y%m%d%H%M%SZ"))) - _log.debug("Modified VO members: %s", [m.account.vsc_id for m in changed_members]) - members = { - NEW: set(), - UPDATED: set(), - ERROR: set(), - DONE: set(), - } - newly_processed_vos = set() - - for member in changed_members: - - if member.group in processed_vos[NEW] \ - or member.group in processed_vos[UPDATED] \ - or member.group in newly_processed_vos: - _log.info("Member %s membership was already processed with group %s" % (member.account.vsc_id, member.group)) - members[DONE].add(member) - continue - - try: - ldap_group = VscLdapGroup(member.group.vsc_id) - ldap_group.status - ldap_group.memberUid = [str(m.account.vsc_id) for m in VoMembership.objects.filter(group=member.group)] - except Exception: - _log.warning("Cannot add member %s to group %s" % (member.account.vsc_id, member.group.vsc_id)) - members[ERROR].add(member) - else: - newly_processed_vos.add(member.group) - members[UPDATED].add(member) - _log.info("Processed group %s member %s" % (member.group.vsc_id, member.account.vsc_id)) - - return members - - - def sync_altered_VO(last, now, dry_run=True): - """ - Synchronise altered VOs back to the LDAP. - """ - changed_vos = VirtualOrganisation.objects.filter(modify_timestamp__range=[last, now]) - _log.info("Found %d modified vos in the range %s until %s" % (len(changed_vos), - last.strftime("%Y%m%d%H%M%SZ"), - now.strftime("%Y%m%d%H%M%SZ"))) - _log.debug("Modified VOs: %s", [v.vsc_id for v in changed_vos]) - - vos = { - NEW: set(), - UPDATED: set(), - ERROR: set(), - } - - for vo in changed_vos: - - # Hack to deal with the anomaly that the VO admin actually 'belongs' to multiple VOs, only in the LDAP - # the moderator need not be a member - if vo.vsc_id in settings.VSC.institute_vos.values(): - moderators = ['vsc40024'] - else: - moderators = [str(m.account.vsc_id) for m in VoMembership.objects.filter(moderator=True, group=vo)] - - ldap_attributes = { + if autogroup: + #TODO get autogroup sources from api + ldap_attributes['autogroup'] = [str(s.vsc_id) for s in autogroup.sources.all()] + if vo: + #TODO: add correct ldap attributes for vo + ldap_attributes = { 'cn': str(vo.vsc_id), 'institute': [str(vo.institute.site)], 'gidNumber': ["%d" % (vo.vsc_id_number,)], @@ -457,13 +272,13 @@ def sync_altered_VO(last, now, dry_run=True): #'scratchQuota': [str(scratch_quota)], } - _log.debug("Proposed changes for VO %s: %s", vo.vsc_id, ldap_attributes) - result = add_or_update(VscLdapGroup, vo.vsc_id, ldap_attributes, dry_run) - vos[result].add(vo) + _log.debug("Proposed changes for group %s: %s", group.vsc_id, ldap_attributes) - return vos + result = add_or_update(VscLdapGroup, group.vsc_id, ldap_attributes, dry_run) + groups[result].add(group) + return groups def main(): now = datetime.utcnow().replace(tzinfo=timezone.utc) @@ -518,7 +333,7 @@ def main(): last = datetime.strptime(last_timestamp, "%Y%m%d%H%M%SZ").replace(tzinfo=timezone.utc) syncer = LdapSyncer() - syncer.sync_altered_accounts(last, now, opts.options.dry_run) + altered_accounts, altered_usergroups = syncer.sync_altered_accounts(last, now, opts.options.dry_run) syncer.sync_altered_pubkeys(last, now, opts.options.dry_run) # altered_users = sync_altered_users(last, now, altered_accounts) # FIXME: no modification timestamps here :( @@ -526,16 +341,14 @@ def main(): _log.debug("Altered pubkeys: %s", syncer.altered_pubkeys) # _log.debug("Altered users: %s" % (altered_users,)) - altered_usergroups = sync_altered_user_groups(last, now, opts.options.dry_run) - altered_autogroups = sync_altered_autogroups(opts.options.dry_run) - altered_groups = sync_altered_groups(last, now, opts.options.dry_run) altered_members = sync_altered_group_membership(last, now, altered_groups, opts.options.dry_run) + altered_autogroups = sync_altered_autogroups(altered_members, opts.options.dry_run) + altered_vos = sync_altered_VO(last, now, opts.options.dry_run) altered_vo_members = sync_altered_vo_membership(last, now, altered_vos, opts.options.dry_run) - _log.debug("Altered usergroups: %s" % (altered_usergroups,)) _log.debug("Altered autogroups: %s" % (altered_autogroups,)) _log.debug("Altered groups: %s" % (altered_groups,)) _log.debug("Altered members: %s" % (altered_members,)) From c33eab1d9b5c6be5147f385fe78d6d2906799b02 Mon Sep 17 00:00:00 2001 From: Jens Timmerman Date: Wed, 22 Jun 2016 14:02:56 +0200 Subject: [PATCH 04/26] further progess of wip --- bin/sync_django_ldap.py | 125 +++++++-------------------------- lib/vsc/administration/user.py | 24 +------ 2 files changed, 28 insertions(+), 121 deletions(-) diff --git a/bin/sync_django_ldap.py b/bin/sync_django_ldap.py index b977b495..75d4f904 100644 --- a/bin/sync_django_ldap.py +++ b/bin/sync_django_ldap.py @@ -28,6 +28,8 @@ from vsc.config.base import GENT, ACTIVE, VSC_CONF_DEFAULT_FILENAME from vsc.accountpage.client import AccountpageClient +from vsc.accountpage.wrappers import mkVscAutogroup, mkVscGroup, mkVscAccountPubkey, mkVscUserGroup + from vsc.ldap.configuration import VscConfiguration from vsc.ldap.entities import VscLdapUser, VscLdapGroup @@ -99,63 +101,17 @@ def add_or_update(self, VscLdapKlass, cn, ldap_attributes, dry_run): def get_public_keys(self, vsc_id): """Get a list of public keys for a given vsc id""" - pks = [mkVscAccountPubkey(p) for p in self.client.account[p.vsc_id] if not p.deleted] + #TODO: check deleted syntax + pks = [mkVscAccountPubkey(p) for p in self.client.account[p.vsc_id].pubkey if not p['deleted']] if not pks: pks = [ACCOUNT_WITHOUT_PUBLIC_KEYS_MAGIC_STRING] return pks - def sync_altered_pubkeys(last, dry_run=True): - """ - Remove obsolete public keys from the LDAP and add new public keys to the LDAP. - """ - changed_pubkeys = [mkVscAccountPubkey(p) for p in self.client.account.pubkey.modified[last].get()[1]] - - pubkeys = { - UPDATED: set(), - DONE: set(), - ERROR: set(), - } - - new_pubkeys = [p for p in changed_pubkeys if not p.deleted] - deleted_pubkeys = [p for p in changed_pubkeys if p.deleted] - - _log.warning("Deleted pubkeys %s" % (deleted_pubkeys,)) - _log.debug("New pubkeys %s", new_pubkeys) - - for p in changed_pubkeys: - - if not p.vsc_id: - # This should NOT happen - _log.error("Key %d had no associated user anymore",p) - continue - - try: - account = mkVscAccount(self.client.account[p.vsc_id]) - except HTTPError: - _log.warning("No account for the user %s corresponding to the public key %d" % (p.vsc_id, p)) - continue - - if account in self.processed_accounts[NEW] or account in self.processed_accounts[UPDATED]: - _log.info("Account %s was already processed and has the new set of public keys" % (account.vsc_id,)) - pubkeys[DONE].add(p) - continue - - try: - ldap_user = VscLdapUser(p.vsc_id) - ldap_user.pubkey = [pk.pubkey for pk in pks] - self.processed_accounts[UPDATED].add(account) - pubkeys[UPDATED].add(p) - except Exception: - _log.warning("Cannot add pubkey for account %s to LDAP" % (account.vsc_id,)) - pubkeys[ERROR].add(p) - - self.processed_pubkeys = pubkeys - - def sync_altered_accounts(self, last, dry_run=True): """ Add new users to the LDAP and update altered users. This does not include usergroups. + this does include pubkeys @type last: datetime @return: tuple (new, updated, error) that indicates what accounts were new, changed or could not be altered. """ @@ -177,7 +133,7 @@ def sync_altered_accounts(self, last, dry_run=True): for account in sync_accounts: try: - usergroup = mkUserGroup(client.account[account.vsc_id].usergroup.get()[1]) + usergroup = mkVscUserGroup(client.account[account.vsc_id].usergroup.get()[1]) except HTTPError: _log.error("No corresponding UserGroup for user %s" % (account.vsc_id,)) continue @@ -218,7 +174,7 @@ def sync_altered_accounts(self, last, dry_run=True): return accounts - def sync_altered_groups(last, now, dry_run=True): + def sync_altered_groups(self, last, now, dry_run=True): """ Synchronise altered groups back to LDAP. This also includes usergroups @@ -236,11 +192,14 @@ def sync_altered_groups(last, now, dry_run=True): } for group in changed_groups: - #TDOO: if is VO or autogroup, set autogroup sources or fairshare and scratch/data dirs - autogroup = False vo = False - - + try: + vo = mkVo(self.client.vo[group.vsc_id].get()[1]) + voquota = self.client.vo[group.vsc_id].quota.get()[1] + except HTTPError as err: + # if a 404 occured, the autogroup does not exist, otherwise something else went wrong. + if err.code != 404: + raise ldap_attributes = { 'cn': str(group.vsc_id), @@ -250,26 +209,14 @@ def sync_altered_groups(last, now, dry_run=True): 'memberUid': [str(a['vsc_id') for a in group.members], 'status': [str(group.status)], } - - if autogroup: - #TODO get autogroup sources from api - ldap_attributes['autogroup'] = [str(s.vsc_id) for s in autogroup.sources.all()] if vo: - #TODO: add correct ldap attributes for vo - ldap_attributes = { - 'cn': str(vo.vsc_id), - 'institute': [str(vo.institute.site)], - 'gidNumber': ["%d" % (vo.vsc_id_number,)], - 'moderator': moderators, - 'memberUid': [str(m.account.vsc_id) for m in VoMembership.objects.filter(group=vo)], - 'status': [str(vo.status)], - 'fairshare': ["%d" % (vo.fairshare,)], - 'description': [str(vo.description)], - 'dataDirectory': [str(vo.data_path)], - 'scratchDirectory': [str(vo.scratch_path)], - #TODO: what do here? - #'dataQuota': [str(data_quota)], - #'scratchQuota': [str(scratch_quota)], + ldap_attributes['fairshare'] = ["%d" % (vo.fairshare,)] + ldap_attributes['description'] = [str(vo.description)] + ldap_attributes['dataDirectory'] = [str(vo.data_path)] + ldap_attributes['scratchDirectory'] = [str(vo.scratch_path)] + #TODO: fix quota: have proper api documentation + #ldap_attributes['dataQuota'] = [str(vo_quota[)], + #ldap_attributes['scratchQuota'] = [str(vo_quota[)], } @@ -299,8 +246,8 @@ def main(): last_timestamp = read_timestamp(SYNC_TIMESTAMP_FILENAME) except Exception: _log.warning("Something broke reading the timestamp from %s" % SYNC_TIMESTAMP_FILENAME) - last_timestamp = "201404230000Z" - _log.warning("We will resync from the beginning of the account page era, i.e. %s" % (last_timestamp,)) + last_timestamp = "201604230000Z" + _log.warning("We will resync from a while back : %s" % (last_timestamp,)) _log.info("Using timestamp %s" % (last_timestamp)) @@ -333,34 +280,16 @@ def main(): last = datetime.strptime(last_timestamp, "%Y%m%d%H%M%SZ").replace(tzinfo=timezone.utc) syncer = LdapSyncer() - altered_accounts, altered_usergroups = syncer.sync_altered_accounts(last, now, opts.options.dry_run) - syncer.sync_altered_pubkeys(last, now, opts.options.dry_run) - # altered_users = sync_altered_users(last, now, altered_accounts) # FIXME: no modification timestamps here :( + altered_accounts = syncer.sync_altered_accounts(last, now, opts.options.dry_run) _log.debug("Altered accounts: %s", syncer.processed_accounts) - _log.debug("Altered pubkeys: %s", syncer.altered_pubkeys) - # _log.debug("Altered users: %s" % (altered_users,)) - - altered_groups = sync_altered_groups(last, now, opts.options.dry_run) - altered_members = sync_altered_group_membership(last, now, altered_groups, opts.options.dry_run) - altered_autogroups = sync_altered_autogroups(altered_members, opts.options.dry_run) - - altered_vos = sync_altered_VO(last, now, opts.options.dry_run) - altered_vo_members = sync_altered_vo_membership(last, now, altered_vos, opts.options.dry_run) + altered_groups = syncer.sync_altered_groups(last, now, opts.options.dry_run) - _log.debug("Altered autogroups: %s" % (altered_autogroups,)) _log.debug("Altered groups: %s" % (altered_groups,)) - _log.debug("Altered members: %s" % (altered_members,)) - _log.debug("Altered VOs: %s" % (altered_vos,)) - _log.debug("Altered VO members: %s" % (altered_vo_members,)) if not altered_accounts[ERROR] \ and not altered_groups[ERROR] \ - and not altered_vos[ERROR] \ - and not altered_members[ERROR] \ - and not altered_vo_members[ERROR] \ - and not altered_usergroups[ERROR] \ _log.info("Child process exiting correctly") sys.exit(0) else: @@ -369,10 +298,6 @@ def main(): ["%s: %s\n" % (k, v) for (k, v) in [ ("altered accounts", altered_accounts[ERROR]), ("altered groups", altered_groups[ERROR]), - ("altered vos", altered_vos[ERROR]), - ("altered members", altered_members[ERROR]), - ("altered vo_members", altered_vo_members[ERROR]), - ("altered usergroups", altered_usergroups[ERROR]), ]] )) sys.exit(-1) diff --git a/lib/vsc/administration/user.py b/lib/vsc/administration/user.py index 1062062f..c4aad18d 100644 --- a/lib/vsc/administration/user.py +++ b/lib/vsc/administration/user.py @@ -15,11 +15,6 @@ # """ This file contains the utilities for dealing with users on the VSC. -Original Perl code by Stijn De Weirdt. - -The following actions are available for users: -- add: Add a user. Requires: institute, gecos, mail address, public key -- modify_quota: Change the personal quota for a user (data and scratch only) @author: Stijn De Weirdt (Ghent University) @author: Andy Georges (Ghent University) @@ -34,7 +29,7 @@ from vsc.utils import fancylogger from vsc.accountpage.wrappers import mkVscAccountPerson, mkVscAccountPubkey, mkVscHomeOnScratch, mkVscUserGroup from vsc.accountpage.wrappers import mkVscAccount, mkUserGroup -from vsc.accountpage.wrappers import mkVscGroup, mkVscUserSizeQuota +from vsc.accountpage.wrappers import mkGroup, mkVscUserSizeQuota from vsc.administration.tools import create_stat_directory from vsc.config.base import VSC, Muk, VscStorage, VSC_DATA, VSC_HOME from vsc.config.base import NEW, MODIFIED, MODIFY, ACTIVE @@ -533,7 +528,8 @@ def __setattr__(self, name, value): def update_user_status(user, options, client): """ - Change the status of the user's account in the account page to active. Do the same for the corresponding UserGroup. + Change the status of the user's account in the account page to active. + The usergroup status is always in sync with thte accounts status """ if user.dry_run: log.info("User %s has account status %s. Dry-run, not changing anything", user.user_id, user.account.status) @@ -557,20 +553,6 @@ def update_user_status(user, options, client): log.error("Account %s status was not changed", user.user_id) raise UserStatusUpdateError("Account %s status was not changed, still at %s" % (user.user_id, account.status)) - try: - response_usergroup = client.account[user.user_id].usergroup.patch(body=payload) - except HTTPError, err: - log.error("UserGroup %s status was not changed", user.user_id) - raise UserStatusUpdateError("UserGroup %s status was not changed - received HTTP code %d" % (user.user_id, err.code)) - - else: - usergroup = mkUserGroup(response_usergroup[1]) - if usergroup.status == ACTIVE: - log.info("UserGroup %s status changed to %s" % (user.user_id, ACTIVE)) - else: - log.error("UserGroup %s status was not changed", user.user_id) - raise UserStatusUpdateError("UserGroup %s status was not changed, still at %s" % - (user.user_id, usergroup.status)) def process_users_quota(options, user_quota, storage_name, client): From ac369345fca7585eed048fbd5bcd472cf465472e Mon Sep 17 00:00:00 2001 From: Jens Timmerman Date: Mon, 7 Aug 2017 16:35:14 +0200 Subject: [PATCH 05/26] bumped version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0b19a20a..6d689007 100644 --- a/setup.py +++ b/setup.py @@ -23,7 +23,7 @@ from vsc.install.shared_setup import ag PACKAGE = { - 'version': '0.36.2', + 'version': '1.0.0', 'author': [ag], 'maintainer': [ag], 'tests_require': ['mock'], From 0ca795764a28f894f7881b4a67f1d77ea03d40cd Mon Sep 17 00:00:00 2001 From: Jens Timmerman Date: Mon, 7 Aug 2017 16:40:09 +0200 Subject: [PATCH 06/26] create accountpageclient --- bin/sync_django_ldap.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/bin/sync_django_ldap.py b/bin/sync_django_ldap.py index d4a29a92..6b23c289 100644 --- a/bin/sync_django_ldap.py +++ b/bin/sync_django_ldap.py @@ -60,8 +60,8 @@ def class LdapSyncer(object): This class implements a system for syncing changes from the accountpage api to the vsc ldap """ - def __init__(self): - self.client = AccountpageClient() # TODO: things here + def __init__(self, client): + self.client = client def add_or_update(self, VscLdapKlass, cn, ldap_attributes, dry_run): """ @@ -230,6 +230,7 @@ def main(): options = { 'nagios-check-interval-threshold': NAGIOS_CHECK_INTERVAL_THRESHOLD, 'start-timestamp': ("The timestamp form which to start, otherwise use the cached value", None, "store", None), + 'access_token': ('OAuth2 token identifying the user with the accountpage', None, 'store', None), } opts = ExtendedSimpleOption(options) stats = {} @@ -274,9 +275,10 @@ def main(): _log.info("Now running as %s" % (os.geteuid(),)) except OSError: _log.raiseException("Could not drop privileges") - last = datetime.strptime(last_timestamp, "%Y%m%d%H%M%SZ").replace(tzinfo=timezone.utc) - syncer = LdapSyncer() + + client = AccountpageClient(token=opts.options.access_token) + syncer = LdapSyncer(client) altered_accounts = syncer.sync_altered_accounts(last, now, opts.options.dry_run) _log.debug("Altered accounts: %s", syncer.processed_accounts) From bb460c4db1ff66e5bf8cc3d5e541ba97d51414d1 Mon Sep 17 00:00:00 2001 From: Jens Timmerman Date: Mon, 7 Aug 2017 17:11:47 +0200 Subject: [PATCH 07/26] fixed setting quota --- bin/sync_django_ldap.py | 41 +++++++++++++++++++++++++++++++---------- 1 file changed, 31 insertions(+), 10 deletions(-) diff --git a/bin/sync_django_ldap.py b/bin/sync_django_ldap.py index 6b23c289..d683a2a5 100644 --- a/bin/sync_django_ldap.py +++ b/bin/sync_django_ldap.py @@ -100,7 +100,6 @@ def add_or_update(self, VscLdapKlass, cn, ldap_attributes, dry_run): def get_public_keys(self, vsc_id): """Get a list of public keys for a given vsc id""" - #TODO: check deleted syntax pks = [mkVscAccountPubkey(p) for p in self.client.account[p.vsc_id].pubkey if not p['deleted']] if not pks: pks = [ACCOUNT_WITHOUT_PUBLIC_KEYS_MAGIC_STRING] @@ -132,15 +131,23 @@ def sync_altered_accounts(self, last, dry_run=True): for account in sync_accounts: try: - usergroup = mkVscUserGroup(client.account[account.vsc_id].usergroup.get()[1]) + usergroup = mkVscUserGroup(self.client.account[account.vsc_id].usergroup.get()[1]) except HTTPError: _log.error("No corresponding UserGroup for user %s" % (account.vsc_id,)) continue - try: + try: gecos = str(account.user.person.gecos) except UnicodeEncodeError: gecos = account.person.gecos.encode('ascii', 'ignore') _log.warning("Converting unicode to ascii for gecos resulting in %s", gecos) + quotas = self.client.account[account.vsc_id].quota.get()[1] + account_quota = {} + for quota in quotas: + for stype in ['VSC_HOME', 'VSC_DATA', 'VSC_SCRATCH_DELCATTY'] + if quota['storage']['storage_type'] is stype and quota['fileset'].startswith('vsc'): + account_quota{stype] = quota["hard"] + + public_keys = self.get_public_keys(account.vsc_id) @@ -155,10 +162,9 @@ def sync_altered_accounts(self, last, dry_run=True): 'homeDirectory': [str(account.home_directory)], 'dataDirectory': [str(account.data_directory)], 'scratchDirectory': [str(account.scratch_directory)], - #TODO: fill in - #'homeQuota': ["%d" % (home_quota,)], - #'dataQuota': ["%d" % (data_quota,)], - #'scratchQuota': ["%d" % (scratch_quota,)], + 'homeQuota': ["%d" % account_quota['VSC_HOME']], + 'dataQuota': ["%d" % account_quota['VSC_DATA']], + 'scratchQuota': ["%d" % account_quota['VSC_SCRATCH_DELCATTY']], 'pubkey': public_keys, 'gidNumber': [str(usergroup.vsc_id_number)], 'loginShell': [str(account.login_shell)], @@ -197,6 +203,14 @@ def sync_altered_groups(self, last, now, dry_run=True): # if a 404 occured, the autogroup does not exist, otherwise something else went wrong. if err.code != 404: raise + quotas = self.client.account[group.vsc_id].quota.get()[1] + account_quota = {} + for quota in quotas: + for stype in ['VSC_HOME', 'VSC_DATA', 'VSC_SCRATCH_DELCATTY'] + if quota['storage']['storage_type'] is stype and quota['fileset'].startswith('vsc'): + account_quota{stype] = quota["hard"] + + ldap_attributes = { 'cn': str(group.vsc_id), @@ -207,13 +221,20 @@ def sync_altered_groups(self, last, now, dry_run=True): 'status': [str(group.status)], } if vo: + quotas = self.client.account[group.vsc_id].quota.get()[1] + vo_quota = {} + for quota in quotas: + for stype in ['VSC_DATA', 'VSC_SCRATCH_DELCATTY'] + if quota['storage']['storage_type'] is stype and quota['fileset'].startswith('gvo'): + vo_quota{stype] = quota["hard"] + + ldap_attributes['fairshare'] = ["%d" % (vo.fairshare,)] ldap_attributes['description'] = [str(vo.description)] ldap_attributes['dataDirectory'] = [str(vo.data_path)] ldap_attributes['scratchDirectory'] = [str(vo.scratch_path)] - #TODO: fix quota: have proper api documentation - #ldap_attributes['dataQuota'] = [str(vo_quota[)], - #ldap_attributes['scratchQuota'] = [str(vo_quota[)], + ldap_attributes['dataQuota'] = [vo_quota['VSC_DATA']], + ldap_attributes['scratchQuota'] = [vo_quota['VSC_SCRATCH_DELCATTY']], } _log.debug("Proposed changes for group %s: %s", group.vsc_id, ldap_attributes) From 5f9f6b613291794b7a6899b58c3afa23a9b7a95f Mon Sep 17 00:00:00 2001 From: Jens Timmerman Date: Mon, 7 Aug 2017 17:17:26 +0200 Subject: [PATCH 08/26] added Jenkinsfile --- Jenkinsfile | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 Jenkinsfile diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 00000000..f708ecad --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,14 @@ +#!/usr/bin/env groovy + +node { + stage 'Checkout' + checkout scm + stage 'install dependencies' + sh "wget -O ez_setup.py https://bootstrap.pypa.io/ez_setup.py" + sh "python ez_setup.py --user" + sh "python -m easy_install -U --user vsc-install" + stage 'cleanup' + sh "git clean -fd" + stage 'test' + sh "python setup.py test" +} From 0fc9a76654b79531f465086105ac5f8207aa3b67 Mon Sep 17 00:00:00 2001 From: Jens Timmerman Date: Mon, 7 Aug 2017 17:22:53 +0200 Subject: [PATCH 09/26] mkVscUserGroup -> mkUserGroup --- bin/sync_django_ldap.py | 4 ++-- lib/vsc/administration/user.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/sync_django_ldap.py b/bin/sync_django_ldap.py index d683a2a5..629a3770 100644 --- a/bin/sync_django_ldap.py +++ b/bin/sync_django_ldap.py @@ -28,7 +28,7 @@ from vsc.config.base import GENT, ACTIVE, VSC_CONF_DEFAULT_FILENAME from vsc.accountpage.client import AccountpageClient -from vsc.accountpage.wrappers import mkVscAutogroup, mkVscGroup, mkVscAccountPubkey, mkVscUserGroup +from vsc.accountpage.wrappers import mkVscAutogroup, mkVscGroup, mkVscAccountPubkey, mkUserGroup from vsc.ldap.configuration import VscConfiguration from vsc.ldap.entities import VscLdapUser, VscLdapGroup @@ -131,7 +131,7 @@ def sync_altered_accounts(self, last, dry_run=True): for account in sync_accounts: try: - usergroup = mkVscUserGroup(self.client.account[account.vsc_id].usergroup.get()[1]) + usergroup = mkUserGroup(self.client.account[account.vsc_id].usergroup.get()[1]) except HTTPError: _log.error("No corresponding UserGroup for user %s" % (account.vsc_id,)) continue diff --git a/lib/vsc/administration/user.py b/lib/vsc/administration/user.py index e3e86c35..9b9b8a25 100644 --- a/lib/vsc/administration/user.py +++ b/lib/vsc/administration/user.py @@ -27,7 +27,7 @@ from urllib2 import HTTPError from vsc.utils import fancylogger -from vsc.accountpage.wrappers import mkVscAccountPerson, mkVscAccountPubkey, mkVscHomeOnScratch, mkVscUserGroup +from vsc.accountpage.wrappers import mkVscAccountPerson, mkVscAccountPubkey, mkVscHomeOnScratch from vsc.accountpage.wrappers import mkVscAccount, mkUserGroup from vsc.accountpage.wrappers import mkGroup, mkVscUserSizeQuota from vsc.administration.tools import create_stat_directory From d7ba9e5594d0b449ec528f56373115ef57465564 Mon Sep 17 00:00:00 2001 From: Jens Timmerman Date: Tue, 8 Aug 2017 14:13:47 +0200 Subject: [PATCH 10/26] fixed prospector issues --- bin/sync_django_ldap.py | 51 +++++++++++++++-------------------------- 1 file changed, 18 insertions(+), 33 deletions(-) diff --git a/bin/sync_django_ldap.py b/bin/sync_django_ldap.py index 629a3770..a1e6d6c0 100644 --- a/bin/sync_django_ldap.py +++ b/bin/sync_django_ldap.py @@ -25,10 +25,10 @@ from datetime import datetime, timezone from ldap import LDAPError -from vsc.config.base import GENT, ACTIVE, VSC_CONF_DEFAULT_FILENAME +from vsc.config.base import VSC_CONF_DEFAULT_FILENAME from vsc.accountpage.client import AccountpageClient -from vsc.accountpage.wrappers import mkVscAutogroup, mkVscGroup, mkVscAccountPubkey, mkUserGroup +from vsc.accountpage.wrappers import mkVscAccountPubkey, mkUserGroup from vsc.ldap.configuration import VscConfiguration from vsc.ldap.entities import VscLdapUser, VscLdapGroup @@ -55,7 +55,7 @@ ERROR = 'error' -def class LdapSyncer(object): +class LdapSyncer(object): """ This class implements a system for syncing changes from the accountpage api to the vsc ldap @@ -97,10 +97,9 @@ def add_or_update(self, VscLdapKlass, cn, ldap_attributes, dry_run): return ERROR return UPDATED - def get_public_keys(self, vsc_id): """Get a list of public keys for a given vsc id""" - pks = [mkVscAccountPubkey(p) for p in self.client.account[p.vsc_id].pubkey if not p['deleted']] + pks = [mkVscAccountPubkey(p) for p in self.client.account[p.vsc_id].pubkey if not p['deleted']] if not pks: pks = [ACCOUNT_WITHOUT_PUBLIC_KEYS_MAGIC_STRING] return pks @@ -113,7 +112,7 @@ def sync_altered_accounts(self, last, dry_run=True): @type last: datetime @return: tuple (new, updated, error) that indicates what accounts were new, changed or could not be altered. """ - changed_accounts= [mkVscAccount(a) for a in self.client.account.modified[last].get()[1]] + changed_accounts = [mkVscAccount(a) for a in self.client.account.modified[last].get()[1]] now = datetime.now() accounts = { @@ -143,11 +142,9 @@ def sync_altered_accounts(self, last, dry_run=True): quotas = self.client.account[account.vsc_id].quota.get()[1] account_quota = {} for quota in quotas: - for stype in ['VSC_HOME', 'VSC_DATA', 'VSC_SCRATCH_DELCATTY'] + for stype in ['VSC_HOME', 'VSC_DATA', 'VSC_SCRATCH_DELCATTY']: if quota['storage']['storage_type'] is stype and quota['fileset'].startswith('vsc'): - account_quota{stype] = quota["hard"] - - + account_quota[stype] = quota["hard"] public_keys = self.get_public_keys(account.vsc_id) @@ -172,7 +169,7 @@ def sync_altered_accounts(self, last, dry_run=True): 'researchField': [account.research_field], 'status': [str(account.status)], } - result = add_or_update(VscLdapUser, account.vsc_id, ldap_attributes, dry_run) + result = add_or_update(VscLdapUser, account.vsc_id, ldap_attributes, dry_run) accounts[result].add(account) return accounts @@ -182,7 +179,7 @@ def sync_altered_groups(self, last, now, dry_run=True): Synchronise altered groups back to LDAP. This also includes usergroups """ - changed_groups= [mkGroup(a) for a in self.client.allgroups.modified[last].get()[1]] + changed_groups = [mkGroup(a) for a in self.client.allgroups.modified[last].get()[1]] _log.info("Found %d modified groups in the range %s until %s" % (len(changed_groups), last.strftime("%Y%m%d%H%M%SZ"), @@ -192,7 +189,7 @@ def sync_altered_groups(self, last, now, dry_run=True): NEW: set(), UPDATED: set(), ERROR: set(), - } + } for group in changed_groups: vo = False @@ -203,31 +200,20 @@ def sync_altered_groups(self, last, now, dry_run=True): # if a 404 occured, the autogroup does not exist, otherwise something else went wrong. if err.code != 404: raise - quotas = self.client.account[group.vsc_id].quota.get()[1] - account_quota = {} - for quota in quotas: - for stype in ['VSC_HOME', 'VSC_DATA', 'VSC_SCRATCH_DELCATTY'] - if quota['storage']['storage_type'] is stype and quota['fileset'].startswith('vsc'): - account_quota{stype] = quota["hard"] - - - ldap_attributes = { 'cn': str(group.vsc_id), 'institute': [str(group.institute)], 'gidNumber': ["%d" % (group.vsc_id_number,)], 'moderator': [str(m['vsc_id']) for m in group.moderators], - 'memberUid': [str(a['vsc_id') for a in group.members], + 'memberUid': [str(a['vsc_id']) for a in group.members], 'status': [str(group.status)], } if vo: - quotas = self.client.account[group.vsc_id].quota.get()[1] - vo_quota = {} - for quota in quotas: - for stype in ['VSC_DATA', 'VSC_SCRATCH_DELCATTY'] - if quota['storage']['storage_type'] is stype and quota['fileset'].startswith('gvo'): - vo_quota{stype] = quota["hard"] - + vo_quota = {} + for quota in voquota: + for stype in ['VSC_DATA', 'VSC_SCRATCH_DELCATTY']: + if quota['storage']['storage_type'] is stype and quota['fileset'].startswith('gvo'): + vo_quota[stype] = quota["hard"] ldap_attributes['fairshare'] = ["%d" % (vo.fairshare,)] ldap_attributes['description'] = [str(vo.description)] @@ -235,7 +221,6 @@ def sync_altered_groups(self, last, now, dry_run=True): ldap_attributes['scratchDirectory'] = [str(vo.scratch_path)] ldap_attributes['dataQuota'] = [vo_quota['VSC_DATA']], ldap_attributes['scratchQuota'] = [vo_quota['VSC_SCRATCH_DELCATTY']], - } _log.debug("Proposed changes for group %s: %s", group.vsc_id, ldap_attributes) @@ -308,8 +293,8 @@ def main(): _log.debug("Altered groups: %s" % (altered_groups,)) - if not altered_accounts[ERROR] \ - and not altered_groups[ERROR] \ + if not altered_accounts[ynROR] \ + and not altered_groups[ERROR]: _log.info("Child process exiting correctly") sys.exit(0) else: From 82fe8f3e4916ca76c0fbc113fba2fab0f1b18936 Mon Sep 17 00:00:00 2001 From: Jens Timmerman Date: Tue, 8 Aug 2017 14:21:35 +0200 Subject: [PATCH 11/26] fixed pylint issues --- bin/sync_django_ldap.py | 11 ++++++----- lib/vsc/administration/user.py | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/bin/sync_django_ldap.py b/bin/sync_django_ldap.py index a1e6d6c0..0e3bf304 100644 --- a/bin/sync_django_ldap.py +++ b/bin/sync_django_ldap.py @@ -21,6 +21,7 @@ import os import pwd import sys +from urllib2 import HTTPError from datetime import datetime, timezone @@ -28,7 +29,7 @@ from vsc.config.base import VSC_CONF_DEFAULT_FILENAME from vsc.accountpage.client import AccountpageClient -from vsc.accountpage.wrappers import mkVscAccountPubkey, mkUserGroup +from vsc.accountpage.wrappers import mkVscAccount, mkVscAccountPubkey, mkUserGroup, mkGroup, mkVo from vsc.ldap.configuration import VscConfiguration from vsc.ldap.entities import VscLdapUser, VscLdapGroup @@ -99,7 +100,7 @@ def add_or_update(self, VscLdapKlass, cn, ldap_attributes, dry_run): def get_public_keys(self, vsc_id): """Get a list of public keys for a given vsc id""" - pks = [mkVscAccountPubkey(p) for p in self.client.account[p.vsc_id].pubkey if not p['deleted']] + pks = [mkVscAccountPubkey(p) for p in self.client.account[vsc_id].pubkey if not p['deleted']] if not pks: pks = [ACCOUNT_WITHOUT_PUBLIC_KEYS_MAGIC_STRING] return pks @@ -169,7 +170,7 @@ def sync_altered_accounts(self, last, dry_run=True): 'researchField': [account.research_field], 'status': [str(account.status)], } - result = add_or_update(VscLdapUser, account.vsc_id, ldap_attributes, dry_run) + result = self.add_or_update(VscLdapUser, account.vsc_id, ldap_attributes, dry_run) accounts[result].add(account) return accounts @@ -224,7 +225,7 @@ def sync_altered_groups(self, last, now, dry_run=True): _log.debug("Proposed changes for group %s: %s", group.vsc_id, ldap_attributes) - result = add_or_update(VscLdapGroup, group.vsc_id, ldap_attributes, dry_run) + result = self.add_or_update(VscLdapGroup, group.vsc_id, ldap_attributes, dry_run) groups[result].add(group) return groups @@ -293,7 +294,7 @@ def main(): _log.debug("Altered groups: %s" % (altered_groups,)) - if not altered_accounts[ynROR] \ + if not altered_accounts[ERROR] \ and not altered_groups[ERROR]: _log.info("Child process exiting correctly") sys.exit(0) diff --git a/lib/vsc/administration/user.py b/lib/vsc/administration/user.py index 9b9b8a25..3fe82aad 100644 --- a/lib/vsc/administration/user.py +++ b/lib/vsc/administration/user.py @@ -27,7 +27,7 @@ from urllib2 import HTTPError from vsc.utils import fancylogger -from vsc.accountpage.wrappers import mkVscAccountPerson, mkVscAccountPubkey, mkVscHomeOnScratch +from vsc.accountpage.wrappers import mkVscAccountPubkey, mkVscHomeOnScratch from vsc.accountpage.wrappers import mkVscAccount, mkUserGroup from vsc.accountpage.wrappers import mkGroup, mkVscUserSizeQuota from vsc.administration.tools import create_stat_directory From f5eb0f1a8d2247341cabcf3e94fe2ff271cfa68e Mon Sep 17 00:00:00 2001 From: Jens Timmerman Date: Tue, 8 Aug 2017 14:33:08 +0200 Subject: [PATCH 12/26] fix timezone import --- bin/sync_django_ldap.py | 3 ++- setup.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/bin/sync_django_ldap.py b/bin/sync_django_ldap.py index 0e3bf304..c8f11134 100644 --- a/bin/sync_django_ldap.py +++ b/bin/sync_django_ldap.py @@ -23,7 +23,8 @@ import sys from urllib2 import HTTPError -from datetime import datetime, timezone +from datetime import datetime +import pytz as timezone from ldap import LDAPError from vsc.config.base import VSC_CONF_DEFAULT_FILENAME diff --git a/setup.py b/setup.py index 6d689007..33928532 100644 --- a/setup.py +++ b/setup.py @@ -36,6 +36,7 @@ 'vsc-ldap-extension >= 1.3', 'vsc-utils >= 1.4.4', 'lockfile >= 0.9.1', + 'pytz', # following dependencies are intentionally not declared until #11 is addressed #'vsc-postgres', #'django', From f52fa9329265255b819d294018c96b4698df6c67 Mon Sep 17 00:00:00 2001 From: Jens Timmerman Date: Tue, 8 Aug 2017 15:15:44 +0200 Subject: [PATCH 13/26] use one 'now' to have less chances of race conditions --- bin/sync_django_ldap.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/bin/sync_django_ldap.py b/bin/sync_django_ldap.py index c8f11134..d43f9f8a 100644 --- a/bin/sync_django_ldap.py +++ b/bin/sync_django_ldap.py @@ -64,6 +64,7 @@ class LdapSyncer(object): """ def __init__(self, client): self.client = client + self.now = datetime.utcnow().replace(tzinfo=timezone.utc) def add_or_update(self, VscLdapKlass, cn, ldap_attributes, dry_run): """ @@ -115,7 +116,6 @@ def sync_altered_accounts(self, last, dry_run=True): @return: tuple (new, updated, error) that indicates what accounts were new, changed or could not be altered. """ changed_accounts = [mkVscAccount(a) for a in self.client.account.modified[last].get()[1]] - now = datetime.now() accounts = { NEW: set(), @@ -127,7 +127,7 @@ def sync_altered_accounts(self, last, dry_run=True): _log.info("Found %d modified accounts in the range %s until %s" % (len(sync_accounts), last.strftime("%Y%m%d%H%M%SZ"), - now.strftime("%Y%m%d%H%M%SZ"))) + self.now.strftime("%Y%m%d%H%M%SZ"))) _log.debug("Modified accounts: %s", [a.vsc_id for a in sync_accounts]) for account in sync_accounts: @@ -176,7 +176,7 @@ def sync_altered_accounts(self, last, dry_run=True): return accounts - def sync_altered_groups(self, last, now, dry_run=True): + def sync_altered_groups(self, last, dry_run=True): """ Synchronise altered groups back to LDAP. This also includes usergroups @@ -185,7 +185,7 @@ def sync_altered_groups(self, last, now, dry_run=True): _log.info("Found %d modified groups in the range %s until %s" % (len(changed_groups), last.strftime("%Y%m%d%H%M%SZ"), - now.strftime("%Y%m%d%H%M%SZ"))) + self.now.strftime("%Y%m%d%H%M%SZ"))) _log.debug("Modified groups: %s", [g.vsc_id for g in changed_groups]) groups = { NEW: set(), @@ -233,7 +233,6 @@ def sync_altered_groups(self, last, now, dry_run=True): def main(): - now = datetime.utcnow().replace(tzinfo=timezone.utc) options = { 'nagios-check-interval-threshold': NAGIOS_CHECK_INTERVAL_THRESHOLD, @@ -287,11 +286,11 @@ def main(): client = AccountpageClient(token=opts.options.access_token) syncer = LdapSyncer(client) - altered_accounts = syncer.sync_altered_accounts(last, now, opts.options.dry_run) + altered_accounts = syncer.sync_altered_accounts(last, opts.options.dry_run) _log.debug("Altered accounts: %s", syncer.processed_accounts) - altered_groups = syncer.sync_altered_groups(last, now, opts.options.dry_run) + altered_groups = syncer.sync_altered_groups(last, opts.options.dry_run) _log.debug("Altered groups: %s" % (altered_groups,)) @@ -319,7 +318,7 @@ def main(): if not result: if not opts.options.start_timestamp: - (_, ldap_timestamp) = convert_timestamp(now) + (_, ldap_timestamp) = convert_timestamp(self.now) if not opts.options.dry_run: write_timestamp(SYNC_TIMESTAMP_FILENAME, ldap_timestamp) else: From e866ed6ea1c29484046a27c19f11b034855bb279 Mon Sep 17 00:00:00 2001 From: Jens Timmerman Date: Tue, 8 Aug 2017 15:38:45 +0200 Subject: [PATCH 14/26] convert timestamp to epoch for less troubles during summer-winter time conversions --- bin/sync_django_ldap.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/sync_django_ldap.py b/bin/sync_django_ldap.py index d43f9f8a..5c87a490 100644 --- a/bin/sync_django_ldap.py +++ b/bin/sync_django_ldap.py @@ -251,7 +251,7 @@ def main(): last_timestamp = read_timestamp(SYNC_TIMESTAMP_FILENAME) except Exception: _log.warning("Something broke reading the timestamp from %s" % SYNC_TIMESTAMP_FILENAME) - last_timestamp = "201604230000Z" + last_timestamp = "201707230000Z" _log.warning("We will resync from a while back : %s" % (last_timestamp,)) _log.info("Using timestamp %s" % (last_timestamp)) @@ -282,10 +282,10 @@ def main(): _log.info("Now running as %s" % (os.geteuid(),)) except OSError: _log.raiseException("Could not drop privileges") - last = datetime.strptime(last_timestamp, "%Y%m%d%H%M%SZ").replace(tzinfo=timezone.utc) client = AccountpageClient(token=opts.options.access_token) syncer = LdapSyncer(client) + last = int((datetime.strptime(last_timestamp, "%Y%m%d%H%M%SZ") - datetime(1970, 1, 1)).total_seconds()) altered_accounts = syncer.sync_altered_accounts(last, opts.options.dry_run) _log.debug("Altered accounts: %s", syncer.processed_accounts) From f4eaa5b056339f539c7f481d8724dc54b676bc83 Mon Sep 17 00:00:00 2001 From: Jens Timmerman Date: Tue, 8 Aug 2017 16:38:50 +0200 Subject: [PATCH 15/26] added some debug info --- bin/sync_django_ldap.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/bin/sync_django_ldap.py b/bin/sync_django_ldap.py index 5c87a490..02c4a3da 100644 --- a/bin/sync_django_ldap.py +++ b/bin/sync_django_ldap.py @@ -102,9 +102,12 @@ def add_or_update(self, VscLdapKlass, cn, ldap_attributes, dry_run): def get_public_keys(self, vsc_id): """Get a list of public keys for a given vsc id""" - pks = [mkVscAccountPubkey(p) for p in self.client.account[vsc_id].pubkey if not p['deleted']] + pks = self.client.account[vsc_id].pubkey + _log.debug('got pks from accountpage: %s', pks) + pks = [mkVscAccountPubkey(p) for p in pks if not p['deleted']] if not pks: pks = [ACCOUNT_WITHOUT_PUBLIC_KEYS_MAGIC_STRING] + _log.warning('account without public keys: %s', vsc_id) return pks def sync_altered_accounts(self, last, dry_run=True): @@ -126,7 +129,7 @@ def sync_altered_accounts(self, last, dry_run=True): sync_accounts = list(changed_accounts) _log.info("Found %d modified accounts in the range %s until %s" % (len(sync_accounts), - last.strftime("%Y%m%d%H%M%SZ"), + datetime.fromtimestamp(last).strftime("%Y%m%d%H%M%SZ"), self.now.strftime("%Y%m%d%H%M%SZ"))) _log.debug("Modified accounts: %s", [a.vsc_id for a in sync_accounts]) @@ -137,16 +140,18 @@ def sync_altered_accounts(self, last, dry_run=True): _log.error("No corresponding UserGroup for user %s" % (account.vsc_id,)) continue try: - gecos = str(account.user.person.gecos) + gecos = str(account.person.gecos) except UnicodeEncodeError: gecos = account.person.gecos.encode('ascii', 'ignore') _log.warning("Converting unicode to ascii for gecos resulting in %s", gecos) + _log.debug('fetching quota') quotas = self.client.account[account.vsc_id].quota.get()[1] account_quota = {} for quota in quotas: for stype in ['VSC_HOME', 'VSC_DATA', 'VSC_SCRATCH_DELCATTY']: if quota['storage']['storage_type'] is stype and quota['fileset'].startswith('vsc'): account_quota[stype] = quota["hard"] + _log.debug('fetching public key') public_keys = self.get_public_keys(account.vsc_id) @@ -184,7 +189,7 @@ def sync_altered_groups(self, last, dry_run=True): changed_groups = [mkGroup(a) for a in self.client.allgroups.modified[last].get()[1]] _log.info("Found %d modified groups in the range %s until %s" % (len(changed_groups), - last.strftime("%Y%m%d%H%M%SZ"), + datetime.fromtimestamp(last).strftime("%Y%m%d%H%M%SZ"), self.now.strftime("%Y%m%d%H%M%SZ"))) _log.debug("Modified groups: %s", [g.vsc_id for g in changed_groups]) groups = { From ecadc39884d6e6ad516c00d8aa2c46056d816647 Mon Sep 17 00:00:00 2001 From: Jens Timmerman Date: Tue, 8 Aug 2017 16:40:47 +0200 Subject: [PATCH 16/26] actually get public keys --- bin/sync_django_ldap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/sync_django_ldap.py b/bin/sync_django_ldap.py index 02c4a3da..56c370ae 100644 --- a/bin/sync_django_ldap.py +++ b/bin/sync_django_ldap.py @@ -102,7 +102,7 @@ def add_or_update(self, VscLdapKlass, cn, ldap_attributes, dry_run): def get_public_keys(self, vsc_id): """Get a list of public keys for a given vsc id""" - pks = self.client.account[vsc_id].pubkey + pks = self.client.account[vsc_id].pubkey.get()[1] _log.debug('got pks from accountpage: %s', pks) pks = [mkVscAccountPubkey(p) for p in pks if not p['deleted']] if not pks: From b1a40e46a0beba1d917364fcaa5c2ac956ea1a32 Mon Sep 17 00:00:00 2001 From: Jens Timmerman Date: Tue, 8 Aug 2017 17:20:11 +0200 Subject: [PATCH 17/26] fixes, don't store vo quota in ldap, pick storage type instead of storage name --- bin/sync_django_ldap.py | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/bin/sync_django_ldap.py b/bin/sync_django_ldap.py index 56c370ae..1c147017 100644 --- a/bin/sync_django_ldap.py +++ b/bin/sync_django_ldap.py @@ -72,7 +72,7 @@ def add_or_update(self, VscLdapKlass, cn, ldap_attributes, dry_run): @return: NEW, UPDATED or ERROR, depending on the operation and its result. """ - ldap_entries = self.vscldapclass.lookup(CnFilter(cn)) + ldap_entries = VscLdapKlass.lookup(CnFilter(cn)) if not ldap_entries: # add the entry _log.debug("add new entry %s %s with the attributes %s", VscLdapKlass.__name__, cn, ldap_attributes) @@ -148,8 +148,8 @@ def sync_altered_accounts(self, last, dry_run=True): quotas = self.client.account[account.vsc_id].quota.get()[1] account_quota = {} for quota in quotas: - for stype in ['VSC_HOME', 'VSC_DATA', 'VSC_SCRATCH_DELCATTY']: - if quota['storage']['storage_type'] is stype and quota['fileset'].startswith('vsc'): + for stype in ['home', 'data', 'scratch']: + if quota['storage']['storage_type'] == stype and quota['fileset'].startswith('vsc'): account_quota[stype] = quota["hard"] _log.debug('fetching public key') @@ -166,9 +166,9 @@ def sync_altered_accounts(self, last, dry_run=True): 'homeDirectory': [str(account.home_directory)], 'dataDirectory': [str(account.data_directory)], 'scratchDirectory': [str(account.scratch_directory)], - 'homeQuota': ["%d" % account_quota['VSC_HOME']], - 'dataQuota': ["%d" % account_quota['VSC_DATA']], - 'scratchQuota': ["%d" % account_quota['VSC_SCRATCH_DELCATTY']], + 'homeQuota': "%d" % account_quota['home'], + 'dataQuota': "%d" % account_quota['data'], + 'scratchQuota': "%d" % account_quota['scratch'], 'pubkey': public_keys, 'gidNumber': [str(usergroup.vsc_id_number)], 'loginShell': [str(account.login_shell)], @@ -202,7 +202,8 @@ def sync_altered_groups(self, last, dry_run=True): vo = False try: vo = mkVo(self.client.vo[group.vsc_id].get()[1]) - voquota = self.client.vo[group.vsc_id].quota.get()[1] + # dont set vo quota in ldap, it's not usefull + # voquota = self.client.vo[group.vsc_id].quota.get()[1] except HTTPError as err: # if a 404 occured, the autogroup does not exist, otherwise something else went wrong. if err.code != 404: @@ -216,18 +217,10 @@ def sync_altered_groups(self, last, dry_run=True): 'status': [str(group.status)], } if vo: - vo_quota = {} - for quota in voquota: - for stype in ['VSC_DATA', 'VSC_SCRATCH_DELCATTY']: - if quota['storage']['storage_type'] is stype and quota['fileset'].startswith('gvo'): - vo_quota[stype] = quota["hard"] - ldap_attributes['fairshare'] = ["%d" % (vo.fairshare,)] ldap_attributes['description'] = [str(vo.description)] ldap_attributes['dataDirectory'] = [str(vo.data_path)] ldap_attributes['scratchDirectory'] = [str(vo.scratch_path)] - ldap_attributes['dataQuota'] = [vo_quota['VSC_DATA']], - ldap_attributes['scratchQuota'] = [vo_quota['VSC_SCRATCH_DELCATTY']], _log.debug("Proposed changes for group %s: %s", group.vsc_id, ldap_attributes) @@ -323,7 +316,7 @@ def main(): if not result: if not opts.options.start_timestamp: - (_, ldap_timestamp) = convert_timestamp(self.now) + (_, ldap_timestamp) = convert_timestamp(datetime.now()) if not opts.options.dry_run: write_timestamp(SYNC_TIMESTAMP_FILENAME, ldap_timestamp) else: From 81a8b629b5e7a7df87bf634921c153b188b2d5f9 Mon Sep 17 00:00:00 2001 From: Jens Timmerman Date: Tue, 8 Aug 2017 18:05:18 +0200 Subject: [PATCH 18/26] make sure to store strings in ldap, and you can't add lists to a set --- bin/sync_django_ldap.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/bin/sync_django_ldap.py b/bin/sync_django_ldap.py index 1c147017..33b83884 100644 --- a/bin/sync_django_ldap.py +++ b/bin/sync_django_ldap.py @@ -169,15 +169,15 @@ def sync_altered_accounts(self, last, dry_run=True): 'homeQuota': "%d" % account_quota['home'], 'dataQuota': "%d" % account_quota['data'], 'scratchQuota': "%d" % account_quota['scratch'], - 'pubkey': public_keys, + 'pubkey': [str(public_keys)], 'gidNumber': [str(usergroup.vsc_id_number)], 'loginShell': [str(account.login_shell)], # 'mukHomeOnScratch': ["FALSE"], # FIXME, see #37 - 'researchField': [account.research_field], + 'researchField': [str(account.research_field[0])], 'status': [str(account.status)], } result = self.add_or_update(VscLdapUser, account.vsc_id, ldap_attributes, dry_run) - accounts[result].add(account) + accounts[result].add(account.vsc_id) return accounts @@ -286,11 +286,11 @@ def main(): last = int((datetime.strptime(last_timestamp, "%Y%m%d%H%M%SZ") - datetime(1970, 1, 1)).total_seconds()) altered_accounts = syncer.sync_altered_accounts(last, opts.options.dry_run) - _log.debug("Altered accounts: %s", syncer.processed_accounts) + _log.debug("Altered accounts: %s", altered_accounts) altered_groups = syncer.sync_altered_groups(last, opts.options.dry_run) - _log.debug("Altered groups: %s" % (altered_groups,)) + _log.debug("Altered groups: %s" % altered_groups) if not altered_accounts[ERROR] \ and not altered_groups[ERROR]: From b0f02c5077464662da49c576da4376ba625d45b8 Mon Sep 17 00:00:00 2001 From: Jens Timmerman Date: Thu, 14 Sep 2017 16:26:59 +0200 Subject: [PATCH 19/26] dict is immutable --- bin/sync_django_ldap.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/bin/sync_django_ldap.py b/bin/sync_django_ldap.py index 33b83884..955886ca 100644 --- a/bin/sync_django_ldap.py +++ b/bin/sync_django_ldap.py @@ -202,8 +202,6 @@ def sync_altered_groups(self, last, dry_run=True): vo = False try: vo = mkVo(self.client.vo[group.vsc_id].get()[1]) - # dont set vo quota in ldap, it's not usefull - # voquota = self.client.vo[group.vsc_id].quota.get()[1] except HTTPError as err: # if a 404 occured, the autogroup does not exist, otherwise something else went wrong. if err.code != 404: @@ -212,8 +210,8 @@ def sync_altered_groups(self, last, dry_run=True): 'cn': str(group.vsc_id), 'institute': [str(group.institute)], 'gidNumber': ["%d" % (group.vsc_id_number,)], - 'moderator': [str(m['vsc_id']) for m in group.moderators], - 'memberUid': [str(a['vsc_id']) for a in group.members], + 'moderator': [str(m) for m in group.moderators], + 'memberUid': [str(a) for a in group.members], 'status': [str(group.status)], } if vo: @@ -225,7 +223,7 @@ def sync_altered_groups(self, last, dry_run=True): _log.debug("Proposed changes for group %s: %s", group.vsc_id, ldap_attributes) result = self.add_or_update(VscLdapGroup, group.vsc_id, ldap_attributes, dry_run) - groups[result].add(group) + groups[result].add(group.vsc_id) return groups From bace2725d6cf9a9cacd04423443681fa1867eb80 Mon Sep 17 00:00:00 2001 From: Jens Timmerman Date: Thu, 14 Sep 2017 16:54:35 +0200 Subject: [PATCH 20/26] correctly select user quota --- bin/sync_django_ldap.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/sync_django_ldap.py b/bin/sync_django_ldap.py index 955886ca..7a4abe8f 100644 --- a/bin/sync_django_ldap.py +++ b/bin/sync_django_ldap.py @@ -149,7 +149,8 @@ def sync_altered_accounts(self, last, dry_run=True): account_quota = {} for quota in quotas: for stype in ['home', 'data', 'scratch']: - if quota['storage']['storage_type'] == stype and quota['fileset'].startswith('vsc'): + # only gent sets filesets for vo's, so not gvo is user. (other institutes is empty or "None" + if quota['storage']['storage_type'] == stype and not quota['fileset'].startswith('gvo'): account_quota[stype] = quota["hard"] _log.debug('fetching public key') From f2e0e278dca335b93d80463748ae5348cff34a3a Mon Sep 17 00:00:00 2001 From: Jens Timmerman Date: Thu, 14 Sep 2017 17:11:45 +0200 Subject: [PATCH 21/26] save institute.site, save pubkeys in right format --- bin/sync_django_ldap.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/sync_django_ldap.py b/bin/sync_django_ldap.py index 7a4abe8f..6e80f8fb 100644 --- a/bin/sync_django_ldap.py +++ b/bin/sync_django_ldap.py @@ -104,7 +104,7 @@ def get_public_keys(self, vsc_id): """Get a list of public keys for a given vsc id""" pks = self.client.account[vsc_id].pubkey.get()[1] _log.debug('got pks from accountpage: %s', pks) - pks = [mkVscAccountPubkey(p) for p in pks if not p['deleted']] + pks = [str(mkVscAccountPubkey(p).pubkey) for p in pks if not p['deleted']] if not pks: pks = [ACCOUNT_WITHOUT_PUBLIC_KEYS_MAGIC_STRING] _log.warning('account without public keys: %s', vsc_id) @@ -161,7 +161,7 @@ def sync_altered_accounts(self, last, dry_run=True): 'uidNumber': ["%s" % (account.vsc_id_number,)], 'gecos': [gecos], 'mail': [str(account.email)], - 'institute': [str(account.person.institute)], + 'institute': [str(account.person.institute['site'])], 'instituteLogin': [str(account.person.institute_login)], 'uid': [str(account.vsc_id)], 'homeDirectory': [str(account.home_directory)], @@ -170,7 +170,7 @@ def sync_altered_accounts(self, last, dry_run=True): 'homeQuota': "%d" % account_quota['home'], 'dataQuota': "%d" % account_quota['data'], 'scratchQuota': "%d" % account_quota['scratch'], - 'pubkey': [str(public_keys)], + 'pubkey': public_keys, 'gidNumber': [str(usergroup.vsc_id_number)], 'loginShell': [str(account.login_shell)], # 'mukHomeOnScratch': ["FALSE"], # FIXME, see #37 @@ -209,7 +209,7 @@ def sync_altered_groups(self, last, dry_run=True): raise ldap_attributes = { 'cn': str(group.vsc_id), - 'institute': [str(group.institute)], + 'institute': [str(group.institute.site)], 'gidNumber': ["%d" % (group.vsc_id_number,)], 'moderator': [str(m) for m in group.moderators], 'memberUid': [str(a) for a in group.members], From 07ad64b275d299511acfbe38018225ee315196e8 Mon Sep 17 00:00:00 2001 From: Jens Timmerman Date: Tue, 19 Sep 2017 16:45:42 +0200 Subject: [PATCH 22/26] move ldapsyncer to it's own lib to prepare for more tests --- bin/sync_django_ldap.py | 189 +------------------------- lib/vsc/administration/ldapsync.py | 209 +++++++++++++++++++++++++++++ setup.py | 3 +- 3 files changed, 214 insertions(+), 187 deletions(-) create mode 100644 lib/vsc/administration/ldapsync.py diff --git a/bin/sync_django_ldap.py b/bin/sync_django_ldap.py index 6e80f8fb..1caff139 100644 --- a/bin/sync_django_ldap.py +++ b/bin/sync_django_ldap.py @@ -21,20 +21,16 @@ import os import pwd import sys -from urllib2 import HTTPError from datetime import datetime -import pytz as timezone -from ldap import LDAPError from vsc.config.base import VSC_CONF_DEFAULT_FILENAME from vsc.accountpage.client import AccountpageClient -from vsc.accountpage.wrappers import mkVscAccount, mkVscAccountPubkey, mkUserGroup, mkGroup, mkVo + +from vsc.administration.ldapsync import LdapSyncer, ERROR from vsc.ldap.configuration import VscConfiguration -from vsc.ldap.entities import VscLdapUser, VscLdapGroup -from vsc.ldap.filters import CnFilter from vsc.ldap.timestamp import convert_timestamp, read_timestamp, write_timestamp from vsc.ldap.utils import LdapQuery from vsc.utils import fancylogger @@ -45,189 +41,10 @@ NAGIOS_CHECK_INTERVAL_THRESHOLD = 15 * 60 # 15 minutes SYNC_TIMESTAMP_FILENAME = "/var/cache/%s.timestamp" % (NAGIOS_HEADER) -ACCOUNT_WITHOUT_PUBLIC_KEYS_MAGIC_STRING = "THIS ACCOUNT HAS NO VALID PUBLIC KEYS" - fancylogger.setLogLevelInfo() fancylogger.logToScreen(True) _log = fancylogger.getLogger(NAGIOS_HEADER) -DONE = 'done' -NEW = 'new' -UPDATED = 'updated' -ERROR = 'error' - - -class LdapSyncer(object): - """ - This class implements a system for syncing changes from the accountpage api - to the vsc ldap - """ - def __init__(self, client): - self.client = client - self.now = datetime.utcnow().replace(tzinfo=timezone.utc) - - def add_or_update(self, VscLdapKlass, cn, ldap_attributes, dry_run): - """ - Perform the update in LDAP for the given vsc.ldap.entitities class, cn and the ldap attributes. - - @return: NEW, UPDATED or ERROR, depending on the operation and its result. - """ - ldap_entries = VscLdapKlass.lookup(CnFilter(cn)) - if not ldap_entries: - # add the entry - _log.debug("add new entry %s %s with the attributes %s", VscLdapKlass.__name__, cn, ldap_attributes) - - if not dry_run: - try: - entry = VscLdapKlass(cn) - entry.add(ldap_attributes) - _log.info("Added a new user %s to LDAP" % (cn,)) - except LDAPError: - _log.warning("Could not add %s %s to LDAP" % (VscLdapKlass.__name__, cn,)) - return ERROR - return NEW - else: - ldap_entries[0].status - _log.debug("update existing entry %s %s with the attributes %s -- old entry: %s", - VscLdapKlass.__name__, cn, ldap_attributes, ldap_entries[0].ldap_info) - - if not dry_run: - try: - ldap_entries[0].modify_ldap(ldap_attributes) - _log.info("Modified %s %s in LDAP" % (VscLdapKlass.__name__, cn,)) - except LDAPError: - _log.warning("Could not add %s %s to LDAP" % (VscLdapKlass.__name__, cn,)) - return ERROR - return UPDATED - - def get_public_keys(self, vsc_id): - """Get a list of public keys for a given vsc id""" - pks = self.client.account[vsc_id].pubkey.get()[1] - _log.debug('got pks from accountpage: %s', pks) - pks = [str(mkVscAccountPubkey(p).pubkey) for p in pks if not p['deleted']] - if not pks: - pks = [ACCOUNT_WITHOUT_PUBLIC_KEYS_MAGIC_STRING] - _log.warning('account without public keys: %s', vsc_id) - return pks - - def sync_altered_accounts(self, last, dry_run=True): - """ - Add new users to the LDAP and update altered users. This does not include usergroups. - - this does include pubkeys - @type last: datetime - @return: tuple (new, updated, error) that indicates what accounts were new, changed or could not be altered. - """ - changed_accounts = [mkVscAccount(a) for a in self.client.account.modified[last].get()[1]] - - accounts = { - NEW: set(), - UPDATED: set(), - ERROR: set(), - } - - sync_accounts = list(changed_accounts) - - _log.info("Found %d modified accounts in the range %s until %s" % (len(sync_accounts), - datetime.fromtimestamp(last).strftime("%Y%m%d%H%M%SZ"), - self.now.strftime("%Y%m%d%H%M%SZ"))) - _log.debug("Modified accounts: %s", [a.vsc_id for a in sync_accounts]) - - for account in sync_accounts: - try: - usergroup = mkUserGroup(self.client.account[account.vsc_id].usergroup.get()[1]) - except HTTPError: - _log.error("No corresponding UserGroup for user %s" % (account.vsc_id,)) - continue - try: - gecos = str(account.person.gecos) - except UnicodeEncodeError: - gecos = account.person.gecos.encode('ascii', 'ignore') - _log.warning("Converting unicode to ascii for gecos resulting in %s", gecos) - _log.debug('fetching quota') - quotas = self.client.account[account.vsc_id].quota.get()[1] - account_quota = {} - for quota in quotas: - for stype in ['home', 'data', 'scratch']: - # only gent sets filesets for vo's, so not gvo is user. (other institutes is empty or "None" - if quota['storage']['storage_type'] == stype and not quota['fileset'].startswith('gvo'): - account_quota[stype] = quota["hard"] - _log.debug('fetching public key') - - public_keys = self.get_public_keys(account.vsc_id) - - ldap_attributes = { - 'cn': str(account.vsc_id), - 'uidNumber': ["%s" % (account.vsc_id_number,)], - 'gecos': [gecos], - 'mail': [str(account.email)], - 'institute': [str(account.person.institute['site'])], - 'instituteLogin': [str(account.person.institute_login)], - 'uid': [str(account.vsc_id)], - 'homeDirectory': [str(account.home_directory)], - 'dataDirectory': [str(account.data_directory)], - 'scratchDirectory': [str(account.scratch_directory)], - 'homeQuota': "%d" % account_quota['home'], - 'dataQuota': "%d" % account_quota['data'], - 'scratchQuota': "%d" % account_quota['scratch'], - 'pubkey': public_keys, - 'gidNumber': [str(usergroup.vsc_id_number)], - 'loginShell': [str(account.login_shell)], - # 'mukHomeOnScratch': ["FALSE"], # FIXME, see #37 - 'researchField': [str(account.research_field[0])], - 'status': [str(account.status)], - } - result = self.add_or_update(VscLdapUser, account.vsc_id, ldap_attributes, dry_run) - accounts[result].add(account.vsc_id) - - return accounts - - def sync_altered_groups(self, last, dry_run=True): - """ - Synchronise altered groups back to LDAP. - This also includes usergroups - """ - changed_groups = [mkGroup(a) for a in self.client.allgroups.modified[last].get()[1]] - - _log.info("Found %d modified groups in the range %s until %s" % (len(changed_groups), - datetime.fromtimestamp(last).strftime("%Y%m%d%H%M%SZ"), - self.now.strftime("%Y%m%d%H%M%SZ"))) - _log.debug("Modified groups: %s", [g.vsc_id for g in changed_groups]) - groups = { - NEW: set(), - UPDATED: set(), - ERROR: set(), - } - - for group in changed_groups: - vo = False - try: - vo = mkVo(self.client.vo[group.vsc_id].get()[1]) - except HTTPError as err: - # if a 404 occured, the autogroup does not exist, otherwise something else went wrong. - if err.code != 404: - raise - ldap_attributes = { - 'cn': str(group.vsc_id), - 'institute': [str(group.institute.site)], - 'gidNumber': ["%d" % (group.vsc_id_number,)], - 'moderator': [str(m) for m in group.moderators], - 'memberUid': [str(a) for a in group.members], - 'status': [str(group.status)], - } - if vo: - ldap_attributes['fairshare'] = ["%d" % (vo.fairshare,)] - ldap_attributes['description'] = [str(vo.description)] - ldap_attributes['dataDirectory'] = [str(vo.data_path)] - ldap_attributes['scratchDirectory'] = [str(vo.scratch_path)] - - _log.debug("Proposed changes for group %s: %s", group.vsc_id, ldap_attributes) - - result = self.add_or_update(VscLdapGroup, group.vsc_id, ldap_attributes, dry_run) - groups[result].add(group.vsc_id) - - return groups - def main(): @@ -285,7 +102,7 @@ def main(): last = int((datetime.strptime(last_timestamp, "%Y%m%d%H%M%SZ") - datetime(1970, 1, 1)).total_seconds()) altered_accounts = syncer.sync_altered_accounts(last, opts.options.dry_run) - _log.debug("Altered accounts: %s", altered_accounts) + _log.debug("Altered accounts: %s", altered_accounts) altered_groups = syncer.sync_altered_groups(last, opts.options.dry_run) diff --git a/lib/vsc/administration/ldapsync.py b/lib/vsc/administration/ldapsync.py new file mode 100644 index 00000000..be4b559c --- /dev/null +++ b/lib/vsc/administration/ldapsync.py @@ -0,0 +1,209 @@ +# -*- coding: latin-1 -*- +# +# Copyright 2013-2017 Ghent University +# +# This file is part of vsc-administration, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# the Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/hpcugent/vsc-administration +# +# All rights reserved. +# +""" +This module contains tools to sync accountpage users to the vsc ldap +""" +from __future__ import absolute_import + +from urllib2 import HTTPError + +import pytz as timezone +from datetime import datetime + +import logging + +from ldap import LDAPError + +from vsc.accountpage.wrappers import mkVscAccount, mkUserGroup, mkGroup, mkVo + +from vsc.ldap.entities import VscLdapUser, VscLdapGroup + +from vsc.ldap.filters import CnFilter + +ACCOUNT_WITHOUT_PUBLIC_KEYS_MAGIC_STRING = "THIS ACCOUNT HAS NO VALID PUBLIC KEYS" + +DONE = 'done' +NEW = 'new' +UPDATED = 'updated' +ERROR = 'error' + + +class LdapSyncer(object): + """ + This class implements a system for syncing changes from the accountpage api + to the vsc ldap + """ + def __init__(self, client): + """ + Create an ldap syncer, requires a RestClient client to get the information from + (typically AccountpageClient) + """ + self.client = client + self.now = datetime.utcnow().replace(tzinfo=timezone.utc) + + def add_or_update(self, VscLdapKlass, cn, ldap_attributes, dry_run): + """ + Perform the update in LDAP for the given vsc.ldap.entitities class, cn and the ldap attributes. + + @return: NEW, UPDATED or ERROR, depending on the operation and its result. + """ + ldap_entries = VscLdapKlass.lookup(CnFilter(cn)) + if not ldap_entries: + # add the entry + logging.debug("add new entry %s %s with the attributes %s", VscLdapKlass.__name__, cn, ldap_attributes) + + if not dry_run: + try: + entry = VscLdapKlass(cn) + entry.add(ldap_attributes) + logging.info("Added a new user %s to LDAP" % (cn,)) + except LDAPError: + logging.warning("Could not add %s %s to LDAP" % (VscLdapKlass.__name__, cn,)) + return ERROR + return NEW + else: + ldap_entries[0].status + logging.debug("update existing entry %s %s with the attributes %s -- old entry: %s", + VscLdapKlass.__name__, cn, ldap_attributes, ldap_entries[0].ldap_info) + + if not dry_run: + try: + ldap_entries[0].modify_ldap(ldap_attributes) + logging.info("Modified %s %s in LDAP" % (VscLdapKlass.__name__, cn,)) + except LDAPError: + logging.warning("Could not add %s %s to LDAP" % (VscLdapKlass.__name__, cn,)) + return ERROR + return UPDATED + + def sync_altered_accounts(self, last, dry_run=True): + """ + Add new users to the LDAP and update altered users. This does not include usergroups. + + this does include pubkeys + @type last: datetime + @return: tuple (new, updated, error) that indicates what accounts were new, changed or could not be altered. + """ + changed_accounts = [mkVscAccount(a) for a in self.client.account.modified[last].get()[1]] + + accounts = { + NEW: set(), + UPDATED: set(), + ERROR: set(), + } + + sync_accounts = list(changed_accounts) + + logging.info("Found %d modified accounts in the range %s until %s" % (len(sync_accounts), + datetime.fromtimestamp(last).strftime("%Y%m%d%H%M%SZ"), + self.now.strftime("%Y%m%d%H%M%SZ"))) + logging.debug("Modified accounts: %s", [a.vsc_id for a in sync_accounts]) + + for account in sync_accounts: + try: + usergroup = mkUserGroup(self.client.account[account.vsc_id].usergroup.get()[1]) + except HTTPError: + logging.error("No corresponding UserGroup for user %s" % (account.vsc_id,)) + continue + try: + gecos = str(account.person.gecos) + except UnicodeEncodeError: + gecos = account.person.gecos.encode('ascii', 'ignore') + logging.warning("Converting unicode to ascii for gecos resulting in %s", gecos) + logging.debug('fetching quota') + quotas = self.client.account[account.vsc_id].quota.get()[1] + account_quota = {} + for quota in quotas: + for stype in ['home', 'data', 'scratch']: + # only gent sets filesets for vo's, so not gvo is user. (other institutes is empty or "None" + if quota['storage']['storage_type'] == stype and not quota['fileset'].startswith('gvo'): + account_quota[stype] = quota["hard"] + logging.debug('fetching public key') + + public_keys = [str(x.pubkey) for x in self.client.get_public_keys(account.vsc_id)] + if not public_keys: + public_keys = [ACCOUNT_WITHOUT_PUBLIC_KEYS_MAGIC_STRING] + + ldap_attributes = { + 'cn': str(account.vsc_id), + 'uidNumber': ["%s" % (account.vsc_id_number,)], + 'gecos': [gecos], + 'mail': [str(account.email)], + 'institute': [str(account.person.institute['site'])], + 'instituteLogin': [str(account.person.institute_login)], + 'uid': [str(account.vsc_id)], + 'homeDirectory': [str(account.home_directory)], + 'dataDirectory': [str(account.data_directory)], + 'scratchDirectory': [str(account.scratch_directory)], + 'homeQuota': ["%d" % account_quota['home']], + 'dataQuota': ["%d" % account_quota['data']], + 'scratchQuota': ["%d" % account_quota['scratch']], + 'pubkey': public_keys, + 'gidNumber': [str(usergroup.vsc_id_number)], + 'loginShell': [str(account.login_shell)], + # 'mukHomeOnScratch': ["FALSE"], # FIXME, see #37 + 'researchField': [str(account.research_field[0])], + 'status': [str(account.status)], + } + result = self.add_or_update(VscLdapUser, account.vsc_id, ldap_attributes, dry_run) + accounts[result].add(account.vsc_id) + + return accounts + + def sync_altered_groups(self, last, dry_run=True): + """ + Synchronise altered groups back to LDAP. + This also includes usergroups + """ + changed_groups = [mkGroup(a) for a in self.client.allgroups.modified[last].get()[1]] + + logging.info("Found %d modified groups in the range %s until %s" % (len(changed_groups), + datetime.fromtimestamp(last).strftime("%Y%m%d%H%M%SZ"), + self.now.strftime("%Y%m%d%H%M%SZ"))) + logging.debug("Modified groups: %s", [g.vsc_id for g in changed_groups]) + groups = { + NEW: set(), + UPDATED: set(), + ERROR: set(), + } + + for group in changed_groups: + vo = False + try: + vo = mkVo(self.client.vo[group.vsc_id].get()[1]) + except HTTPError as err: + # if a 404 occured, the group is not an VO, so we skip this. Otherwise something else went wrong. + if err.code != 404: + raise + ldap_attributes = { + 'cn': str(group.vsc_id), + 'institute': [str(group.institute.site)], + 'gidNumber': ["%d" % (group.vsc_id_number,)], + 'moderator': [str(m) for m in group.moderators], + 'memberUid': [str(a) for a in group.members], + 'status': [str(group.status)], + } + if vo: + ldap_attributes['fairshare'] = ["%d" % (vo.fairshare,)] + ldap_attributes['description'] = [str(vo.description)] + ldap_attributes['dataDirectory'] = [str(vo.data_path)] + ldap_attributes['scratchDirectory'] = [str(vo.scratch_path)] + + logging.debug("Proposed changes for group %s: %s", group.vsc_id, ldap_attributes) + + result = self.add_or_update(VscLdapGroup, group.vsc_id, ldap_attributes, dry_run) + groups[result].add(group.vsc_id) + + return groups diff --git a/setup.py b/setup.py index 8aa307b6..913202ec 100644 --- a/setup.py +++ b/setup.py @@ -28,11 +28,12 @@ 'maintainer': [ag], 'tests_require': ['mock'], 'install_requires': [ - 'vsc-accountpage-clients >= 0.7', + 'vsc-accountpage-clients >= 0.9.0', 'vsc-base >= 2.4.16', 'vsc-config >= 1.31.2', 'vsc-filesystems >= 0.19', 'vsc-ldap >= 1.1', + 'python-ldap', 'vsc-ldap-extension >= 1.3', 'vsc-utils >= 1.4.4', 'lockfile >= 0.9.1', From eb4e5568243b47a1bdb39939be64b29aa5307b5a Mon Sep 17 00:00:00 2001 From: Jens Timmerman Date: Tue, 19 Sep 2017 18:21:44 +0200 Subject: [PATCH 23/26] add testcasses for ldap syncer --- lib/vsc/administration/ldapsync.py | 25 ++++------- test/ldapsync.py | 68 ++++++++++++++++++++++++++++++ test/user.py | 8 ++-- 3 files changed, 81 insertions(+), 20 deletions(-) create mode 100644 test/ldapsync.py diff --git a/lib/vsc/administration/ldapsync.py b/lib/vsc/administration/ldapsync.py index be4b559c..9615e5ab 100644 --- a/lib/vsc/administration/ldapsync.py +++ b/lib/vsc/administration/ldapsync.py @@ -96,16 +96,13 @@ def sync_altered_accounts(self, last, dry_run=True): @type last: datetime @return: tuple (new, updated, error) that indicates what accounts were new, changed or could not be altered. """ - changed_accounts = [mkVscAccount(a) for a in self.client.account.modified[last].get()[1]] - + sync_accounts = [mkVscAccount(a) for a in self.client.account.modified[last].get()[1]] accounts = { NEW: set(), UPDATED: set(), ERROR: set(), } - sync_accounts = list(changed_accounts) - logging.info("Found %d modified accounts in the range %s until %s" % (len(sync_accounts), datetime.fromtimestamp(last).strftime("%Y%m%d%H%M%SZ"), self.now.strftime("%Y%m%d%H%M%SZ"))) @@ -122,14 +119,6 @@ def sync_altered_accounts(self, last, dry_run=True): except UnicodeEncodeError: gecos = account.person.gecos.encode('ascii', 'ignore') logging.warning("Converting unicode to ascii for gecos resulting in %s", gecos) - logging.debug('fetching quota') - quotas = self.client.account[account.vsc_id].quota.get()[1] - account_quota = {} - for quota in quotas: - for stype in ['home', 'data', 'scratch']: - # only gent sets filesets for vo's, so not gvo is user. (other institutes is empty or "None" - if quota['storage']['storage_type'] == stype and not quota['fileset'].startswith('gvo'): - account_quota[stype] = quota["hard"] logging.debug('fetching public key') public_keys = [str(x.pubkey) for x in self.client.get_public_keys(account.vsc_id)] @@ -147,16 +136,20 @@ def sync_altered_accounts(self, last, dry_run=True): 'homeDirectory': [str(account.home_directory)], 'dataDirectory': [str(account.data_directory)], 'scratchDirectory': [str(account.scratch_directory)], - 'homeQuota': ["%d" % account_quota['home']], - 'dataQuota': ["%d" % account_quota['data']], - 'scratchQuota': ["%d" % account_quota['scratch']], 'pubkey': public_keys, 'gidNumber': [str(usergroup.vsc_id_number)], 'loginShell': [str(account.login_shell)], - # 'mukHomeOnScratch': ["FALSE"], # FIXME, see #37 'researchField': [str(account.research_field[0])], 'status': [str(account.status)], } + logging.debug('fetching quota') + quotas = self.client.account[account.vsc_id].quota.get()[1] + for quota in quotas: + for stype in ['home', 'data', 'scratch']: + # only gent sets filesets for vo's, so not gvo is user. (other institutes is empty or "None" + if quota['storage']['storage_type'] == stype and not quota['fileset'].startswith('gvo'): + ldap_attributes['%sQuota' % stype] = ["%d" % quota["hard"]] + result = self.add_or_update(VscLdapUser, account.vsc_id, ldap_attributes, dry_run) accounts[result].add(account.vsc_id) diff --git a/test/ldapsync.py b/test/ldapsync.py new file mode 100644 index 00000000..0510df33 --- /dev/null +++ b/test/ldapsync.py @@ -0,0 +1,68 @@ +# +# Copyright 2015-2017 Ghent University +# +# This file is part of vsc-administration, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# the Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/hpcugent/vsc-administration +# +# All rights reserved. +# +""" +Tests for vsc.administration.user + +@author: Andy Georges (Ghent University) +@author: Jens Timmerman (Ghent University) +""" +import mock + +import vsc + +from vsc.install.testing import TestCase +from vsc.accountpage.wrappers import mkVscAccountPubkey, mkVscAccount + +from vsc.administration.ldapsync import LdapSyncer, UPDATED +from vsc.ldap.entities import VscLdapUser + +from .user import test_account_1, test_usergroup_1, test_pubkeys_1 + +test_quota = [ + { + "user": "vsc10018", + "storage": { + "institute": "brussel", + "name": "VSC_HOME", + "storage_type": "home" + }, + "fileset": "None", + "hard": 5242880 + } +] + + +class LDAPSyncerTest(TestCase): + """ + Tests for the LDAP syncer that sync account page information to the vsc ldap. + """ + + @mock.patch.object(vsc.administration.ldapsync.LdapSyncer, 'add_or_update') + def test_sync_altered_accounts(self, mock_add_or_update): + """Test the sync_altered accounts function""" + mock_client = mock.MagicMock() + test_account = mkVscAccount(test_account_1) + mock_client.account[test_account.vsc_id] = mock.MagicMock() + mock_client.account.modified[1].get.return_value = (200, [test_account_1]) + mock_client.account[test_account.vsc_id].usergroup.get.return_value = (200, test_usergroup_1) + mock_client.get_public_keys.return_value = [mkVscAccountPubkey(p) for p in test_pubkeys_1] + mock_client.account[test_account.vsc_id].quota.get.return_value = (200, test_quota) + + mock_add_or_update.return_value = UPDATED + ldapsyncer = LdapSyncer(mock_client) + accounts = ldapsyncer.sync_altered_accounts(1) + self.assertEqual(accounts, {'error': set([]), 'new': set([]), 'updated': set([test_account.vsc_id])}) + ldap_attrs = {'status': ['active'], 'dataDirectory': ['/user/data/gent/vsc400/vsc40075'], 'cn': 'vsc40075', 'homeQuota': ['5242880'], 'loginShell': ['/bin/bash'], 'uidNumber': ['2540075'], 'gidNumber': ['2540075'], 'instituteLogin': ['foobar'], 'uid': ['vsc40075'], 'scratchDirectory': ['/user/scratch/gent/vsc400/vsc40075'], 'institute': ['gent'], 'researchField': ['Bollocks'], 'gecos': ['Foo Bar'], 'homeDirectory': ['/user/home/gent/vsc400/vsc40075'], 'mail': ['foobar@ugent.be'], 'pubkey': ['pubkey1', 'pubkey2']} + mock_add_or_update.assert_called_with(VscLdapUser, test_account.vsc_id, ldap_attrs, True) diff --git a/test/user.py b/test/user.py index 384f4360..fa4e818f 100644 --- a/test/user.py +++ b/test/user.py @@ -13,11 +13,11 @@ # All rights reserved. # """ -Tests for vsc.administration.vo +Tests for vsc.administration.user @author: Andy Georges (Ghent University) +@author: Jens Timmerman (Ghent University) """ -import logging import mock from collections import namedtuple @@ -25,7 +25,7 @@ import vsc.administration.user as user from vsc.accountpage.wrappers import mkVscAccount, mkVscHomeOnScratch, mkUserGroup, mkGroup -from vsc.accountpage.wrappers import mkVscAccountPubkey, mkVscUserSizeQuota +from vsc.accountpage.wrappers import mkVscAccountPubkey from vsc.config.base import VSC_DATA, VSC_HOME, VSC_SCRATCH_PHANPY, VSC_SCRATCH_DELCATTY from vsc.install.testing import TestCase @@ -445,7 +445,7 @@ def test_create_data_dir_tier2_user(self, @mock.patch.object(user.VscTier2AccountpageUser, '_grouping_scratch_path') @mock.patch.object(user.VscTier2AccountpageUser, '_create_grouping_fileset') @mock.patch.object(user.VscTier2AccountpageUser, '_create_user_dir') - def test_create_data_dir_tier2_user(self, + def test_create_data_dir_tier2_user_2(self, mock_create_user_dir, mock_create_grouping_fileset, mock_grouping_scratch_path, From 4243095f5a51ea1ce12159eade75ceb2979fff14 Mon Sep 17 00:00:00 2001 From: Jens Timmerman Date: Wed, 20 Sep 2017 11:57:41 +0200 Subject: [PATCH 24/26] added tests for group/vo --- lib/vsc/administration/ldapsync.py | 2 +- test/ldapsync.py | 55 ++++++++++++++++++++++++++++-- 2 files changed, 53 insertions(+), 4 deletions(-) diff --git a/lib/vsc/administration/ldapsync.py b/lib/vsc/administration/ldapsync.py index 9615e5ab..b30ae7e1 100644 --- a/lib/vsc/administration/ldapsync.py +++ b/lib/vsc/administration/ldapsync.py @@ -182,7 +182,7 @@ def sync_altered_groups(self, last, dry_run=True): raise ldap_attributes = { 'cn': str(group.vsc_id), - 'institute': [str(group.institute.site)], + 'institute': [str(group.institute['site'])], 'gidNumber': ["%d" % (group.vsc_id_number,)], 'moderator': [str(m) for m in group.moderators], 'memberUid': [str(a) for a in group.members], diff --git a/test/ldapsync.py b/test/ldapsync.py index 0510df33..e6989283 100644 --- a/test/ldapsync.py +++ b/test/ldapsync.py @@ -20,13 +20,15 @@ """ import mock +from urllib2 import HTTPError + import vsc from vsc.install.testing import TestCase -from vsc.accountpage.wrappers import mkVscAccountPubkey, mkVscAccount +from vsc.accountpage.wrappers import mkVscAccountPubkey, mkVscAccount, mkGroup from vsc.administration.ldapsync import LdapSyncer, UPDATED -from vsc.ldap.entities import VscLdapUser +from vsc.ldap.entities import VscLdapUser, VscLdapGroup from .user import test_account_1, test_usergroup_1, test_pubkeys_1 @@ -43,6 +45,25 @@ } ] +test_vo_1 = { + "vsc_id": "gvo00003", + "status": "active", + "vsc_id_number": 2640010, + "institute": { + "site": "gent" + }, + "fairshare": 100, + "data_path": "/user/data/gent/gvo000/gvo00003", + "scratch_path": "/user/scratch/gent/gvo000/gvo00003", + "description": "VO", + "members": [ + "vsc40075", + ], + "moderators": [ + "vsc40075" + ] +} + class LDAPSyncerTest(TestCase): """ @@ -64,5 +85,33 @@ def test_sync_altered_accounts(self, mock_add_or_update): ldapsyncer = LdapSyncer(mock_client) accounts = ldapsyncer.sync_altered_accounts(1) self.assertEqual(accounts, {'error': set([]), 'new': set([]), 'updated': set([test_account.vsc_id])}) - ldap_attrs = {'status': ['active'], 'dataDirectory': ['/user/data/gent/vsc400/vsc40075'], 'cn': 'vsc40075', 'homeQuota': ['5242880'], 'loginShell': ['/bin/bash'], 'uidNumber': ['2540075'], 'gidNumber': ['2540075'], 'instituteLogin': ['foobar'], 'uid': ['vsc40075'], 'scratchDirectory': ['/user/scratch/gent/vsc400/vsc40075'], 'institute': ['gent'], 'researchField': ['Bollocks'], 'gecos': ['Foo Bar'], 'homeDirectory': ['/user/home/gent/vsc400/vsc40075'], 'mail': ['foobar@ugent.be'], 'pubkey': ['pubkey1', 'pubkey2']} + ldap_attrs = {'status': ['active'], 'dataDirectory': ['/user/data/gent/vsc400/vsc40075'], 'cn': 'vsc40075', 'homeQuota': ['5242880'], 'loginShell': ['/bin/bash'], 'uidNumber': ['2540075'], 'gidNumber': ['2540075'], 'instituteLogin': ['foobar'], 'uid': ['vsc40075'], 'scratchDirectory': ['/user/scratch/gent/vsc400/vsc40075'], 'institute': ['gent'], 'researchField': ['Bollocks'], 'gecos': ['Foo Bar'], 'homeDirectory': ['/user/home/gent/vsc400/vsc40075'], 'mail': ['foobar@ugent.be'], 'pubkey': ['pubkey1', 'pubkey2']} mock_add_or_update.assert_called_with(VscLdapUser, test_account.vsc_id, ldap_attrs, True) + + @mock.patch.object(vsc.administration.ldapsync.LdapSyncer, 'add_or_update') + def test_sync_altered_groups(self, mock_add_or_update): + """Test the sync_altered accounts function""" + mock_client = mock.MagicMock() + test_group = mkGroup(test_vo_1) + mock_client.allgroups.modified[1].get.return_value = (200, [test_vo_1]) + mock_client.vo[test_group.vsc_id].get.return_value = (200, test_vo_1) + + mock_add_or_update.return_value = UPDATED + ldapsyncer = LdapSyncer(mock_client) + groups = ldapsyncer.sync_altered_groups(1) + self.assertEqual(groups, {'error': set([]), 'new': set([]), 'updated': set([test_group.vsc_id])}) + + ldap_attrs = {'status': ['active'], 'scratchDirectory': ['/user/scratch/gent/gvo000/gvo00003'], + 'dataDirectory': ['/user/data/gent/gvo000/gvo00003'], 'cn': 'gvo00003', 'institute': ['gent'], + 'memberUid': ['vsc40075'], 'moderator': ['vsc40075'], 'gidNumber': ['2640010'], + 'fairshare': ['100'], 'description': ['VO']} + mock_add_or_update.assert_called_with(VscLdapGroup, test_group.vsc_id, ldap_attrs, True) + + # should actually give a 404 in reallity, but use this to pretend it's not a vo + test_group = mkGroup(test_usergroup_1) + mock_client.allgroups.modified[1].get.return_value = (200, [test_usergroup_1]) + mock_client.vo[test_group.vsc_id].get.side_effect = HTTPError(mock.Mock(status=404), 'not found') + groups = ldapsyncer.sync_altered_groups(1) + self.assertEqual(groups, {'error': set([]), 'new': set([]), 'updated': set([test_group.vsc_id])}) + ldap_attrs = {'status': ['active'], 'cn': 'vsc40075', 'gidNumber': ['2540075'], 'institute': ['gent']} + mock_add_or_update.assert_called_with(VscLdapGroup, test_group.vsc_id, ldap_attrs, True) From 60cca95fbe0ada4de0131c5b63a8bab801275af3 Mon Sep 17 00:00:00 2001 From: Jens Timmerman Date: Wed, 20 Sep 2017 14:52:44 +0200 Subject: [PATCH 25/26] fix httperror on vo --- test/ldapsync.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/ldapsync.py b/test/ldapsync.py index e6989283..181b55a2 100644 --- a/test/ldapsync.py +++ b/test/ldapsync.py @@ -107,11 +107,12 @@ def test_sync_altered_groups(self, mock_add_or_update): 'fairshare': ['100'], 'description': ['VO']} mock_add_or_update.assert_called_with(VscLdapGroup, test_group.vsc_id, ldap_attrs, True) - # should actually give a 404 in reallity, but use this to pretend it's not a vo + # raise a 404 error on getting the VO to trigger a non vo group add instead of a vo add as above test_group = mkGroup(test_usergroup_1) mock_client.allgroups.modified[1].get.return_value = (200, [test_usergroup_1]) - mock_client.vo[test_group.vsc_id].get.side_effect = HTTPError(mock.Mock(status=404), 'not found') + mock_client.vo[test_group.vsc_id].get.side_effect = HTTPError("mock_url", 404, "Not Found", "mock_headers", None) groups = ldapsyncer.sync_altered_groups(1) self.assertEqual(groups, {'error': set([]), 'new': set([]), 'updated': set([test_group.vsc_id])}) - ldap_attrs = {'status': ['active'], 'cn': 'vsc40075', 'gidNumber': ['2540075'], 'institute': ['gent']} + ldap_attrs = {'status': ['active'], 'cn': 'vsc40075', 'institute': ['gent'], 'memberUid': ['vsc40075'], + 'moderator': ['vsc40075'], 'gidNumber': ['2540075']} mock_add_or_update.assert_called_with(VscLdapGroup, test_group.vsc_id, ldap_attrs, True) From 6aebdc0cb288d03b03da7fb2aad6a0136b8de8dc Mon Sep 17 00:00:00 2001 From: Jens Timmerman Date: Mon, 25 Sep 2017 13:30:25 +0200 Subject: [PATCH 26/26] get token from config file --- bin/sync_django_ldap.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bin/sync_django_ldap.py b/bin/sync_django_ldap.py index 1caff139..876b6144 100644 --- a/bin/sync_django_ldap.py +++ b/bin/sync_django_ldap.py @@ -53,6 +53,8 @@ def main(): 'start-timestamp': ("The timestamp form which to start, otherwise use the cached value", None, "store", None), 'access_token': ('OAuth2 token identifying the user with the accountpage', None, 'store', None), } + # get access_token from conf file + ExtendedSimpleOption.CONFIGFILES_INIT = ['/etc/account_page.conf'] opts = ExtendedSimpleOption(options) stats = {}