From bbdd0aa6ab66c5113f08168c6a37547cb04eb561 Mon Sep 17 00:00:00 2001 From: Jens Timmerman Date: Mon, 25 Sep 2017 14:40:07 +0200 Subject: [PATCH 1/5] use our own setup.cfg --- setup.cfg | 19 +++++++++---------- setup.py | 16 ++++------------ 2 files changed, 13 insertions(+), 22 deletions(-) diff --git a/setup.cfg b/setup.cfg index 62655bc3..79685ae0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,16 +1,15 @@ [bdist_rpm] -requires = vsc-accountpage-clients >= 0.9.0 - vsc-base >= 2.4.16 - vsc-config >= 1.31.2 - vsc-filesystems >= 0.19 - vsc-ldap >= 1.1 - python-ldap - vsc-ldap-extension >= 1.3 - vsc-utils >= 1.4.4 - lockfile >= 0.9.1 +requires = python-vsc-accountpage-clients >= 0.9.0 + python-vsc-base >= 2.4.16 + python-vsc-config >= 1.31.2 + python-vsc-filesystems >= 0.19 + python-vsc-ldap >= 1.1 pytz + python-ldap + python-vsc-ldap-extension >= 1.3 + python-vsc-utils >= 1.4.4 + python-lockfile >= 0.9.1 [metadata] - description-file = README.md diff --git a/setup.py b/setup.py index fe460ee0..88483c0c 100644 --- a/setup.py +++ b/setup.py @@ -17,38 +17,30 @@ vsc-administration distribution setup.py @author: Andy Georges (Ghent University) +@author: Jens Timmerman (Ghent University) """ from vsc.install import shared_setup from vsc.install.shared_setup import ag, jt PACKAGE = { - 'version': '1.0.1', + 'version': '1.0.5', 'author': [ag, jt], 'maintainer': [ag, jt], 'tests_require': ['mock'], + 'makesetupcfg': False, # use setup.cfg provided to get pytz instead of python-pytz 'install_requires': [ 'vsc-accountpage-clients >= 0.9.0', 'vsc-base >= 2.4.16', 'vsc-config >= 1.31.2', 'vsc-filesystems >= 0.19', 'vsc-ldap >= 1.1', + 'pytz', 'python-ldap', 'vsc-ldap-extension >= 1.3', 'vsc-utils >= 1.4.4', 'lockfile >= 0.9.1', - 'pytz', - ], - 'dependency_links': [ - "git+https://github.com/hpcugent/vsc-utils.git#egg=vsc-utils-1.8.2", - "git+https://github.com/hpcugent/vsc-filesystems.git#egg=vsc-filesystems-0.30.1", - "git+ssh://github.com/hpcugent/vsc-accountpage-clients.git#egg=vsc-accountpage-clients-0.7", - "git+ssh://github.com/hpcugent/vsc-ldap.git#egg=vsc-ldap-1.4.2", - "git+ssh://github.com/hpcugent/vsc-ldap-extension.git#egg=vsc-ldap-extensions-1.10.2", ], - - - } From daa14e4ef5987f29edafedf77d955c0e1b1f1251 Mon Sep 17 00:00:00 2001 From: Jens Timmerman Date: Mon, 25 Sep 2017 15:25:31 +0200 Subject: [PATCH 2/5] use account_page_url --- bin/sync_django_ldap.py | 3 ++- setup.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/bin/sync_django_ldap.py b/bin/sync_django_ldap.py index 876b6144..7bf2d624 100644 --- a/bin/sync_django_ldap.py +++ b/bin/sync_django_ldap.py @@ -52,6 +52,7 @@ def main(): 'nagios-check-interval-threshold': NAGIOS_CHECK_INTERVAL_THRESHOLD, 'start-timestamp': ("The timestamp form which to start, otherwise use the cached value", None, "store", None), 'access_token': ('OAuth2 token identifying the user with the accountpage', None, 'store', None), + 'account_page_url': ('url for the account page', None, 'store', None), } # get access_token from conf file ExtendedSimpleOption.CONFIGFILES_INIT = ['/etc/account_page.conf'] @@ -99,7 +100,7 @@ def main(): except OSError: _log.raiseException("Could not drop privileges") - client = AccountpageClient(token=opts.options.access_token) + client = AccountpageClient(token=opts.options.access_token, url=opts.options.account_page_url) syncer = LdapSyncer(client) last = int((datetime.strptime(last_timestamp, "%Y%m%d%H%M%SZ") - datetime(1970, 1, 1)).total_seconds()) altered_accounts = syncer.sync_altered_accounts(last, opts.options.dry_run) diff --git a/setup.py b/setup.py index 88483c0c..38583d3a 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ from vsc.install.shared_setup import ag, jt PACKAGE = { - 'version': '1.0.5', + 'version': '1.0.6', 'author': [ag, jt], 'maintainer': [ag, jt], 'tests_require': ['mock'], From 6c4cba1c91e754e07641df8d245f7c1cedb5c914 Mon Sep 17 00:00:00 2001 From: Jens Timmerman Date: Mon, 25 Sep 2017 15:31:28 +0200 Subject: [PATCH 3/5] use account_page_url --- bin/sync_django_ldap.py | 2 +- bin/sync_ugent_vsc_users.py | 6 +++--- setup.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/bin/sync_django_ldap.py b/bin/sync_django_ldap.py index 7bf2d624..456198c7 100644 --- a/bin/sync_django_ldap.py +++ b/bin/sync_django_ldap.py @@ -100,7 +100,7 @@ def main(): except OSError: _log.raiseException("Could not drop privileges") - client = AccountpageClient(token=opts.options.access_token, url=opts.options.account_page_url) + client = AccountpageClient(token=opts.options.access_token, url=opts.options.account_page_url + '/api/') syncer = LdapSyncer(client) last = int((datetime.strptime(last_timestamp, "%Y%m%d%H%M%SZ") - datetime(1970, 1, 1)).total_seconds()) altered_accounts = syncer.sync_altered_accounts(last, opts.options.dry_run) diff --git a/bin/sync_ugent_vsc_users.py b/bin/sync_ugent_vsc_users.py index 381ef1e3..01ae224e 100644 --- a/bin/sync_ugent_vsc_users.py +++ b/bin/sync_ugent_vsc_users.py @@ -14,7 +14,7 @@ # All rights reserved. # """ -This script synchronises the users and VO's from the HPC LDAP to the central +This script synchronises the users and VO's from the HPC account page to the central UGent storage for home and data. For each (active) user, the following tasks are done: @@ -78,7 +78,7 @@ def main(): 'user': ('process users', None, 'store_true', False), 'vo': ('process vos', None, 'store_true', False), 'access_token': ('OAuth2 token to access the account page REST API', None, 'store', None), - 'account_page_url': ('URL of the account page REST API', None, 'store', None) + 'account_page_url': ('URL of the account page where we can find the REST API', None, 'store', None) } opts = ExtendedSimpleOption(options) @@ -86,7 +86,7 @@ def main(): try: now = datetime.utcnow() - client = AccountpageClient(token=opts.options.access_token) + client = AccountpageClient(token=opts.options.access_token, url=opts.options.account_page_url + "/api/") try: last_timestamp = read_timestamp(SYNC_TIMESTAMP_FILENAME) diff --git a/setup.py b/setup.py index 38583d3a..5c036f37 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ from vsc.install.shared_setup import ag, jt PACKAGE = { - 'version': '1.0.6', + 'version': '1.0.7', 'author': [ag, jt], 'maintainer': [ag, jt], 'tests_require': ['mock'], From 3db94fb0c74df4bf70a5e62e5c7d73a92c80f927 Mon Sep 17 00:00:00 2001 From: Jens Timmerman Date: Mon, 25 Sep 2017 15:45:56 +0200 Subject: [PATCH 4/5] set default quote of 1 for scratch, home and data --- lib/vsc/administration/ldapsync.py | 3 +++ setup.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/vsc/administration/ldapsync.py b/lib/vsc/administration/ldapsync.py index b30ae7e1..94e9f00e 100644 --- a/lib/vsc/administration/ldapsync.py +++ b/lib/vsc/administration/ldapsync.py @@ -141,6 +141,9 @@ def sync_altered_accounts(self, last, dry_run=True): 'loginShell': [str(account.login_shell)], 'researchField': [str(account.research_field[0])], 'status': [str(account.status)], + 'homeQuota': ["1"], + 'dataQuota': ["1"], + 'scratchQuota': ["1"], } logging.debug('fetching quota') quotas = self.client.account[account.vsc_id].quota.get()[1] diff --git a/setup.py b/setup.py index 5c036f37..5b27c2f4 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ from vsc.install.shared_setup import ag, jt PACKAGE = { - 'version': '1.0.7', + 'version': '1.0.8', 'author': [ag, jt], 'maintainer': [ag, jt], 'tests_require': ['mock'], From eb4e6600b49061a347cd54db07a513b5a3012dbd Mon Sep 17 00:00:00 2001 From: Jens Timmerman Date: Mon, 25 Sep 2017 16:12:16 +0200 Subject: [PATCH 5/5] set default vsc400024 user --- lib/vsc/administration/ldapsync.py | 5 +++++ setup.py | 2 +- test/ldapsync.py | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/vsc/administration/ldapsync.py b/lib/vsc/administration/ldapsync.py index 94e9f00e..00042986 100644 --- a/lib/vsc/administration/ldapsync.py +++ b/lib/vsc/administration/ldapsync.py @@ -29,6 +29,7 @@ from vsc.accountpage.wrappers import mkVscAccount, mkUserGroup, mkGroup, mkVo +from vsc.config.base import VSC from vsc.ldap.entities import VscLdapUser, VscLdapGroup from vsc.ldap.filters import CnFilter @@ -40,6 +41,7 @@ UPDATED = 'updated' ERROR = 'error' +VSC_CONFIG = VSC() class LdapSyncer(object): """ @@ -196,6 +198,9 @@ def sync_altered_groups(self, last, dry_run=True): ldap_attributes['description'] = [str(vo.description)] ldap_attributes['dataDirectory'] = [str(vo.data_path)] ldap_attributes['scratchDirectory'] = [str(vo.scratch_path)] + # vsc40024 is moderator for all institute vo's + if vo.vsc_id in VSC_CONFIG.institute_vos.values(): + ldap_attributes['moderator'] = ['vsc40024'] logging.debug("Proposed changes for group %s: %s", group.vsc_id, ldap_attributes) diff --git a/setup.py b/setup.py index 5b27c2f4..310a07a1 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ from vsc.install.shared_setup import ag, jt PACKAGE = { - 'version': '1.0.8', + 'version': '1.0.9', 'author': [ag, jt], 'maintainer': [ag, jt], 'tests_require': ['mock'], diff --git a/test/ldapsync.py b/test/ldapsync.py index 181b55a2..f52ccf17 100644 --- a/test/ldapsync.py +++ b/test/ldapsync.py @@ -85,7 +85,7 @@ def test_sync_altered_accounts(self, mock_add_or_update): ldapsyncer = LdapSyncer(mock_client) accounts = ldapsyncer.sync_altered_accounts(1) self.assertEqual(accounts, {'error': set([]), 'new': set([]), 'updated': set([test_account.vsc_id])}) - ldap_attrs = {'status': ['active'], 'dataDirectory': ['/user/data/gent/vsc400/vsc40075'], 'cn': 'vsc40075', 'homeQuota': ['5242880'], 'loginShell': ['/bin/bash'], 'uidNumber': ['2540075'], 'gidNumber': ['2540075'], 'instituteLogin': ['foobar'], 'uid': ['vsc40075'], 'scratchDirectory': ['/user/scratch/gent/vsc400/vsc40075'], 'institute': ['gent'], 'researchField': ['Bollocks'], 'gecos': ['Foo Bar'], 'homeDirectory': ['/user/home/gent/vsc400/vsc40075'], 'mail': ['foobar@ugent.be'], 'pubkey': ['pubkey1', 'pubkey2']} + ldap_attrs = {'status': ['active'], 'scratchDirectory': ['/user/scratch/gent/vsc400/vsc40075'], 'dataDirectory': ['/user/data/gent/vsc400/vsc40075'], 'cn': 'vsc40075', 'homeQuota': ['5242880'], 'institute': ['gent'], 'loginShell': ['/bin/bash'], 'uidNumber': ['2540075'], 'researchField': ['Bollocks'], 'gidNumber': ['2540075'], 'gecos': ['Foo Bar'], 'dataQuota': ['1'], 'homeDirectory': ['/user/home/gent/vsc400/vsc40075'], 'mail': ['foobar@ugent.be'], 'scratchQuota': ['1'], 'pubkey': ['pubkey1', 'pubkey2'], 'instituteLogin': ['foobar'], 'uid': ['vsc40075']} mock_add_or_update.assert_called_with(VscLdapUser, test_account.vsc_id, ldap_attrs, True) @mock.patch.object(vsc.administration.ldapsync.LdapSyncer, 'add_or_update')