From 653bfa7dc23127668137d5486ea40eaf4f7ea14e Mon Sep 17 00:00:00 2001 From: George Taylor Date: Fri, 20 Sep 2024 14:22:45 +0100 Subject: [PATCH] Merge dev into main (#74) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * :bug: Fix incorrect module path for update user roles (#51) * 🚑 Update User Roles: Use sync search and add summary (#52) * :ambulance: Use sync search and add summary for searches and actions * Formatted code with black --line-length 120 --------- Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com> * Better summary for update user roles job (#53) * :ambulance: Use sync search and add summary for searches and actions * Formatted code with black --line-length 120 * better debugging --------- Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com> * Update user.py * :ambulance: Correct logic for matched user sets + role filtering (#55) * :ambulance: correct logic for matched user sets + role filtering * Formatted code with black --line-length 120 * Update role filter logic in user.py --------- Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com> * Fix update user role cmd (#56) * :ambulance: correct logic for matched user sets + role filtering * Formatted code with black --line-length 120 * Update role filter logic in user.py * Formatted code with black --line-length 120 * aliasing * Update user.py --------- Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com> * Fix update user role cmd (#57) * :ambulance: correct logic for matched user sets + role filtering * Formatted code with black --line-length 120 * Update role filter logic in user.py * Formatted code with black --line-length 120 * aliasing * Update user.py * Update user.py * Formatted code with black --line-length 120 --------- Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com> * ✨ Use paged search to support larger data sets (#58) * :ambulance: correct logic for matched user sets + role filtering * Formatted code with black --line-length 120 * Update role filter logic in user.py * Formatted code with black --line-length 120 * aliasing * Update user.py * Update user.py * Formatted code with black --line-length 120 * implement paging because we love ldap :heart: * remove test case + add var for page size * Formatted code with black --line-length 120 --------- Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com> * Bump setuptools from 71.1.0 to 72.0.0 (#60) Bumps [setuptools](https://github.com/pypa/setuptools) from 71.1.0 to 72.0.0. - [Release notes](https://github.com/pypa/setuptools/releases) - [Changelog](https://github.com/pypa/setuptools/blob/main/NEWS.rst) - [Commits](https://github.com/pypa/setuptools/compare/v71.1.0...v72.0.0) --- updated-dependencies: - dependency-name: setuptools dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * :bug: Fix error handling for policy/role recreation (#65) * :bug: error handling for policy/role recreation * :art: formatting * :recycle: Add image build (#66) * :art: formatting * :wrench: Implement Ruff as required check instead of Black creating a commit re-formatting code (#73) * :wrench: Implement Ruff as check instead of Black creating a commit * Update python-checks.yml --------- Signed-off-by: dependabot[bot] Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/format-python.yml | 23 ---- .github/workflows/image-build.yml | 58 +++++++++ .github/workflows/python-checks.yml | 71 +++++++++++ Dockerfile | 12 ++ cli/__init__.py | 32 +++-- cli/database/__init__.py | 4 +- cli/git/__init__.py | 16 ++- cli/ldap_cmds/__init__.py | 8 +- cli/ldap_cmds/rbac.py | 177 ++++++++++++++++++++-------- cli/ldap_cmds/user.py | 99 ++++++++++++---- cli/logger.py | 14 ++- setup.py | 4 +- 12 files changed, 395 insertions(+), 123 deletions(-) delete mode 100644 .github/workflows/format-python.yml create mode 100644 .github/workflows/image-build.yml create mode 100644 .github/workflows/python-checks.yml create mode 100644 Dockerfile diff --git a/.github/workflows/format-python.yml b/.github/workflows/format-python.yml deleted file mode 100644 index f5f84e1..0000000 --- a/.github/workflows/format-python.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: Format Python -on: - pull_request: - types: [ opened, edited, reopened, synchronize, ready_for_review ] - workflow_dispatch: -jobs: - format: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - repository: ${{ github.event.pull_request.head.repo.full_name }} - ref: ${{ github.event.pull_request.head.ref }} - - name: Format code with black - run: | - pip install black - black --line-length 120 . - - name: Commit changes - uses: EndBug/add-and-commit@v9 - with: - default_author: github_actions - message: "Formatted code with black --line-length 120" - add: "." diff --git a/.github/workflows/image-build.yml b/.github/workflows/image-build.yml new file mode 100644 index 0000000..b40917f --- /dev/null +++ b/.github/workflows/image-build.yml @@ -0,0 +1,58 @@ +name: "Image Build" + +on: + workflow_dispatch: + push: + tags: + - 'v*' + +run-name: "Image Build for tag ${{ github.ref_name }}" + +permissions: + packages: write + contents: write + +jobs: + build-and-push: + runs-on: ubuntu-22.04 + + steps: + - name: Checkout Code + uses: actions/checkout@v4 + + - name: Create safe tag for image + id: safe_tag + run: | + echo "SAFE_TAG=$(echo ${{ github.ref_name }} | sed 's/[^a-zA-Z0-9.]/-/g')" >> $GITHUB_OUTPUT + + - name: Set up Docker Buildx + id: setup_buildx + uses: docker/setup-buildx-action@v3 + + - name: Log into ghcr + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push to ghcr + id: build_publish + uses: docker/build-push-action@v6 + with: + context: . + platforms: linux/amd64, linux/arm64 + push: true + tags: ghcr.io/ministryofjustice/hmpps-ldap-automation:${{ steps.safe_tag.outputs.SAFE_TAG }} + build-args: | + VERSION_REF=${{ steps.BumpVersionAndPushTag.outputs.new_tag }} + + - name: Slack failure notification + if: ${{ failure() }} + uses: slackapi/slack-github-action@70cd7be8e40a46e8b0eced40b0de447bdb42f68e # v1.26.0 + with: + payload: | + {"blocks":[{"type": "section","text": {"type": "mrkdwn","text": ":no_entry: Failed GitHub Action:"}},{"type": "section","fields":[{"type": "mrkdwn","text": "*Workflow:*\n<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|${{ github.workflow }}>"},{"type": "mrkdwn","text": "*Job:*\n${{ github.job }}"},{"type": "mrkdwn","text": "*Repo:*\n${{ github.repository }}"}]}]} + env: + SLACK_WEBHOOK_URL: ${{ secrets.PWO_PUBLIC_SLACK_WEBHOOK_URL }} + SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK diff --git a/.github/workflows/python-checks.yml b/.github/workflows/python-checks.yml new file mode 100644 index 0000000..a541558 --- /dev/null +++ b/.github/workflows/python-checks.yml @@ -0,0 +1,71 @@ +name: Ensure formatted code +on: + pull_request: + types: [ opened, edited, reopened, synchronize, ready_for_review ] + workflow_dispatch: + +permissions: + contents: read + pull-requests: write + +jobs: + format_check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Install ruff + run: pip install ruff + - name: Check formatting for Python code + env: + GH_TOKEN: ${{ github.token }} + run: | + set +e + output=$(ruff format --check) + exit_code=$? + if [ $exit_code -eq 0 ]; then + echo "All Python code is properly formatted." + gh pr comment ${{ github.event.pull_request.number }} --body ":white_check_mark: All Python code is properly formatted." + else + echo "$output" + echo "
:rotating_light: Python code is not properly formatted. Click to expand." > output.txt + echo "" >> output.txt + echo '```' >> output.txt + echo "$output" >> output.txt + echo '```' >> output.txt + echo "" >> output.txt + echo '
' >> output.txt + echo "" >> output.txt + echo 'Please run `ruff format` to format the code.' >> output.txt + gh pr comment ${{ github.event.pull_request.number }} --body-file output.txt + exit 1 + fi + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Install ruff + run: pip install ruff + - name: Lint Python code + env: + GH_TOKEN: ${{ github.token }} + run: | + set +e + output=$(ruff check) + exit_code=$? + if [ $exit_code -eq 0 ]; then + echo "No linting errors found." + gh pr comment ${{ github.event.pull_request.number }} --body ":white_check_mark: No linting errors found in Python code." + else + echo "$output" + echo "
:rotating_light: Linting errors found in Python code. Click to expand." > output.txt + echo "" >> output.txt + echo '```' >> output.txt + echo "$output" >> output.txt + echo '```' >> output.txt + echo "" >> output.txt + echo '
' >> output.txt + echo "" >> output.txt + echo 'Tip: You can run `ruff check --fix` to fix automatically fixable errors.' >> output.txt + gh pr comment ${{ github.event.pull_request.number }} --body-file output.txt + exit 1 + fi diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..3afcbe8 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,12 @@ +FROM python:3.10-alpine + +LABEL org.opencontainers.image.source = "https://github.com/ministryofjustice/hmpps-ldap-automation-cli" + +ARG VERSION_REF=main + +# Basic tools for now +RUN apk add --update --no-cache bash ca-certificates git build-base libffi-dev openssl-dev gcc musl-dev gcc g++ linux-headers build-base openldap-dev python3-dev + +RUN python3 -m pip install --upgrade pip && python3 -m pip install git+https://github.com/ministryofjustice/hmpps-ldap-automation-cli.git@${VERSION_REF} + +CMD ["ldap-automation"] diff --git a/cli/__init__.py b/cli/__init__.py index 1a9fcf5..7be218e 100644 --- a/cli/__init__.py +++ b/cli/__init__.py @@ -118,20 +118,26 @@ def update_user_home_areas( help="Remove role from users", is_flag=True, ) -@click.option("-uf", "--user-filter", help="Filter to find users", required=False, default="(objectclass=*)") + +@click.option( + "-uf", + "--user-filter", + help="Filter to find users", + required=False, + default="(objectclass=*)", +) @click.option("--roles-to-filter", help="Roles to filter", required=False, default="*") -def update_user_roles(roles, user_ou, root_dn, add, remove, update_notes, user_note, user_filter, roles_to_filter): - cli.ldap_cmds.user.update_roles( - roles, - user_ou, - root_dn, - add, - remove, - update_notes, - user_note=user_note, - user_filter=user_filter, - roles_to_filter=roles_to_filter, - ) +def update_user_roles( + roles, + user_ou, + root_dn, + add, + remove, + update_notes, + user_note, + user_filter, + roles_to_filter, +): @click.command() diff --git a/cli/database/__init__.py b/cli/database/__init__.py index 0e7dda6..511366b 100644 --- a/cli/database/__init__.py +++ b/cli/database/__init__.py @@ -13,5 +13,7 @@ def connection(): log.debug("Created database connection successfully") return conn except Exception as e: - log.exception(f"Failed to create database connection. An exception of type {type(e).__name__} occurred: {e}") + log.exception( + f"Failed to create database connection. An exception of type {type(e).__name__} occurred: {e}" + ) raise e diff --git a/cli/git/__init__.py b/cli/git/__init__.py index 6b6cf69..8a733c9 100644 --- a/cli/git/__init__.py +++ b/cli/git/__init__.py @@ -36,7 +36,9 @@ def get_access_token( headers=headers, ) except Exception as e: - logging.exception(f"Failed to get access token. An exception of type {type(e).__name__} occurred: {e}") + logging.exception( + f"Failed to get access token. An exception of type {type(e).__name__} occurred: {e}" + ) raise e # extract the token from the response @@ -66,7 +68,9 @@ def get_repo( multi_options=multi_options, ) except Exception as e: - logging.exception(f"Failed to clone repo. An exception of type {type(e).__name__} occurred: {e}") + logging.exception( + f"Failed to clone repo. An exception of type {type(e).__name__} occurred: {e}" + ) raise e # if there is a token, assume auth is required and use the token and auth_type elif token: @@ -79,7 +83,9 @@ def get_repo( multi_options=multi_options, ) except Exception as e: - logging.exception(f"Failed to clone repo. An exception of type {type(e).__name__} occurred: {e}") + logging.exception( + f"Failed to clone repo. An exception of type {type(e).__name__} occurred: {e}" + ) raise e # if there is no token, assume auth is not required and clone without else: @@ -91,5 +97,7 @@ def get_repo( multi_options=multi_options, ) except Exception as e: - logging.exception(f"Failed to clone repo. An exception of type {type(e).__name__} occurred: {e}") + logging.exception( + f"Failed to clone repo. An exception of type {type(e).__name__} occurred: {e}" + ) raise e diff --git a/cli/ldap_cmds/__init__.py b/cli/ldap_cmds/__init__.py index 69110ed..5a33808 100644 --- a/cli/ldap_cmds/__init__.py +++ b/cli/ldap_cmds/__init__.py @@ -5,12 +5,8 @@ # import oracledb -def ldap_connect( - ldap_host, - ldap_user, - ldap_password, -): - server = Server(ldap_host) +def ldap_connect(ldap_host, ldap_port, ldap_user, ldap_password): + server = Server(ldap_host, ldap_port) return Connection( server=server, diff --git a/cli/ldap_cmds/rbac.py b/cli/ldap_cmds/rbac.py index 4d83ec4..cfd1ee4 100644 --- a/cli/ldap_cmds/rbac.py +++ b/cli/ldap_cmds/rbac.py @@ -133,8 +133,12 @@ def context_ldif( # connect to ldap try: - connection = ldap.initialize("ldap://" + env.vars.get("LDAP_HOST")) - connection.simple_bind_s(env.vars.get("LDAP_USER"), env.secrets.get("LDAP_BIND_PASSWORD")) + connection = ldap.initialize( + f"ldap://{env.vars.get('LDAP_HOST')}:{env.vars.get('LDAP_PORT')}" + ) + connection.simple_bind_s( + env.vars.get("LDAP_USER"), env.secrets.get("LDAP_BIND_PASSWORD") + ) except Exception as e: log.exception(f"Failed to connect to ldap") raise e @@ -171,8 +175,12 @@ def group_ldifs( ): # connect to ldap try: - connection = ldap.initialize("ldap://" + env.vars.get("LDAP_HOST")) - connection.simple_bind_s(env.vars.get("LDAP_USER"), env.secrets.get("LDAP_BIND_PASSWORD")) + connection = ldap.initialize( + f"ldap://{env.vars.get('LDAP_HOST')}:{env.vars.get('LDAP_PORT')}" + ) + connection.simple_bind_s( + env.vars.get("LDAP_USER"), env.secrets.get("LDAP_BIND_PASSWORD") + ) except Exception as e: log.exception(f"Failed to connect to ldap") raise e @@ -209,7 +217,10 @@ def group_ldifs( if attributes.get("description"): log.info(f"Updating description for {dn}") try: - connection.modify(dn, [(ldap.MOD_REPLACE, "description", attributes["description"])]) + connection.modify( + dn, + [(ldap.MOD_REPLACE, "description", attributes["description"])], + ) except ldap.ALREADY_EXISTS as already_exists_e: log.info(f"{dn} already exists") log.debug(already_exists_e) @@ -223,37 +234,65 @@ def policy_ldifs( ): # connect to ldap try: - connection = ldap.initialize("ldap://" + env.vars.get("LDAP_HOST")) - connection.simple_bind_s(env.vars.get("LDAP_USER"), env.secrets.get("LDAP_BIND_PASSWORD")) + connection = ldap.initialize( + f"ldap://{env.vars.get('LDAP_HOST')}:{env.vars.get('LDAP_PORT')}" + ) + connection.simple_bind_s( + env.vars.get("LDAP_USER"), env.secrets.get("LDAP_BIND_PASSWORD") + ) except Exception as e: log.exception(f"Failed to connect to ldap") raise e + log.debug("*********************************") + log.debug("STARTING POLICY LDIFS") + log.debug("*********************************") + policy_files = [file for file in rendered_files if "policy" in Path(file).name] # first, delete the policies ldap_config_dict = env.vars.get("LDAP_CONFIG") or ldap_config - policy_tree = "ou=Policies," + ldap_config_dict.get("base_root") + policy_tree = f"ou=Policies,{ldap_config_dict.get('base_root')}" - tree = connection.search_s( - policy_tree, - ldap.SCOPE_SUBTREE, - "(objectClass=*)", - ) - tree.reverse() + log.debug(f"Policy tree: {policy_tree}") - for entry in tree: - try: - log.debug(entry[0]) - connection.delete_ext_s(entry[0], serverctrls=[ldap.controls.simple.ManageDSAITControl()]) - print(f"Deleted {entry[0]}") - except ldap.NO_SUCH_OBJECT as no_such_object_e: - log.info("No such object found, 32") - log.debug(no_such_object_e) + try: + tree = connection.search_s( + policy_tree, + ldap.SCOPE_SUBTREE, + "(objectClass=*)", + ) + tree.reverse() + except ldap.NO_SUCH_OBJECT as no_such_object_e: + log.debug("Entire policy ou does not exist, no need to delete child objects") + tree = None + log.debug("*********************************") + log.debug("DELETING POLICY ENTRIES") + log.debug("*********************************") + + if tree is not None: + for entry in tree: + try: + log.debug(entry[0]) + connection.delete_ext_s( + entry[0], serverctrls=[ldap.controls.simple.ManageDSAITControl()] + ) + print(f"Deleted {entry[0]}") + except ldap.NO_SUCH_OBJECT as no_such_object_e: + log.debug(f"this is the entry {entry}") + log.debug("error deleting entry") + log.info("No such object found, 32") + log.debug(no_such_object_e) + + log.debug("*********************************") + log.debug("RECREATING POLICY ENTRIES") + log.debug("*********************************") # loop through the policy files for file in policy_files: # parse the ldif into dn and record + # + log.debug(f"Reading file {file}") records = ldif.LDIFRecordList(open(file, "rb")) records.parse() @@ -277,6 +316,9 @@ def policy_ldifs( except Exception as e: log.exception(f"Failed to add {dn}... {attributes}") raise e + log.debug("*********************************") + log.debug("FINISHED POLICY LDIFS") + log.debug("*********************************") def role_ldifs( @@ -284,12 +326,20 @@ def role_ldifs( ): # connect to ldap try: - connection = ldap.initialize("ldap://" + env.vars.get("LDAP_HOST")) - connection.simple_bind_s(env.vars.get("LDAP_USER"), env.secrets.get("LDAP_BIND_PASSWORD")) + connection = ldap.initialize( + f"ldap://{env.vars.get('LDAP_HOST')}:{env.vars.get('LDAP_PORT')}" + ) + connection.simple_bind_s( + env.vars.get("LDAP_USER"), env.secrets.get("LDAP_BIND_PASSWORD") + ) except Exception as e: log.exception(f"Failed to connect to ldap") raise e + log.debug("*********************************") + log.debug("STARTING ROLES") + log.debug("*********************************") + role_files = [file for file in rendered_files if "nd_role" in Path(file).name] # first, delete the roles @@ -299,26 +349,39 @@ def role_ldifs( "cn=ndRoleCatalogue," + ldap_config_dict.get("base_users"), "cn=ndRoleGroups," + ldap_config_dict.get("base_users"), ] - for role_tree in role_trees: - tree = connection.search_s( - role_tree, - ldap.SCOPE_SUBTREE, - "(objectClass=*)", - ) - tree.reverse() - for entry in tree: - try: - log.debug(entry[0]) - connection.delete_ext_s(entry[0], serverctrls=[ldap.controls.simple.ManageDSAITControl()]) - print(f"Deleted {entry[0]}") - except ldap.NO_SUCH_OBJECT as no_such_object_e: - log.info("No such object found, 32") - log.debug(no_such_object_e) + for role_tree in role_trees: + try: + tree = connection.search_s( + role_tree, + ldap.SCOPE_SUBTREE, + "(objectClass=*)", + ) + tree.reverse() + except ldap.NO_SUCH_OBJECT as no_such_object_e: + log.debug("Entire role ou does not exist, no need to delete child objects") + tree = None + log.debug("*********************************") + log.debug("DELETING ROLES") + log.debug("*********************************") + if tree is not None: + for entry in tree: + try: + log.debug(entry[0]) + connection.delete_ext_s( + entry[0], + serverctrls=[ldap.controls.simple.ManageDSAITControl()], + ) + print(f"Deleted {entry[0]}") + except ldap.NO_SUCH_OBJECT as no_such_object_e: + log.info("No such object found, 32") + log.debug(no_such_object_e) # ensure boolean values are Uppercase.. this comes from the ansible yml # (not yet implemented, probably not needed) - + log.debug("*********************************") + log.debug("RECREATING ROLES") + log.debug("*********************************") # loop through the role files for file in role_files: # parse the ldif into dn and record @@ -345,6 +408,9 @@ def role_ldifs( except Exception as e: log.exception(f"Failed to add {dn}... {attributes}") raise e + log.debug("*********************************") + log.debug("FINISHED ROLES") + log.debug("*********************************") # not complete!! @@ -354,13 +420,21 @@ def schema_ldifs( ): # connect to ldap try: - connection = ldap.initialize("ldap://" + env.vars.get("LDAP_HOST")) - connection.simple_bind_s(env.vars.get("LDAP_USER"), env.secrets.get("LDAP_BIND_PASSWORD")) + connection = ldap.initialize( + f"ldap://{env.vars.get('LDAP_HOST')}:{env.vars.get('LDAP_PORT')}" + ) + connection.simple_bind_s( + env.vars.get("LDAP_USER"), env.secrets.get("LDAP_BIND_PASSWORD") + ) except Exception as e: log.exception(f"Failed to connect to ldap") raise e - schema_files = [file for file in rendered_files if "delius.ldif" or "pwm.ldif" in Path(file).name] + schema_files = [ + file + for file in rendered_files + if "delius.ldif" or "pwm.ldif" in Path(file).name + ] # loop through the schema files for file in schema_files: @@ -390,8 +464,12 @@ def user_ldifs( ): # connect to ldap try: - connection = ldap.initialize("ldap://" + env.vars.get("LDAP_HOST")) - connection.simple_bind_s(env.vars.get("LDAP_USER"), env.secrets.get("LDAP_BIND_PASSWORD")) + connection = ldap.initialize( + f"ldap://{env.vars.get('LDAP_HOST')}:{env.vars.get('LDAP_PORT')}" + ) + connection.simple_bind_s( + env.vars.get("LDAP_USER"), env.secrets.get("LDAP_BIND_PASSWORD") + ) except Exception as e: log.exception(f"Failed to connect to ldap") raise e @@ -422,7 +500,10 @@ def user_ldifs( for entry in tree: try: log.debug(entry[0]) - connection.delete_ext_s(entry[0], serverctrls=[ldap.controls.simple.ManageDSAITControl()]) + connection.delete_ext_s( + entry[0], + serverctrls=[ldap.controls.simple.ManageDSAITControl()], + ) print(f"Deleted {entry[0]}") except ldap.NO_SUCH_OBJECT as no_such_object_e: log.info("No such object found, 32") @@ -515,7 +596,9 @@ def main( f"{clone_path}/**/*", recursive=True, ) - if Path(file).is_file() and Path(file).name.endswith(".ldif") or Path(file).name.endswith(".j2") + if Path(file).is_file() + and Path(file).name.endswith(".ldif") + or Path(file).name.endswith(".j2") ] prep_for_templating(files) diff --git a/cli/ldap_cmds/user.py b/cli/ldap_cmds/user.py index 2ad4a02..17ad0f7 100644 --- a/cli/ldap_cmds/user.py +++ b/cli/ldap_cmds/user.py @@ -46,13 +46,12 @@ def change_home_areas( log.info(f"Updating user home areas from {old_home_area} to {new_home_area}") ldap_connection = ldap_connect( env.vars.get("LDAP_HOST"), + env.vars.get("LDAP_PORT", 389), env.vars.get("LDAP_USER"), env.secrets.get("LDAP_BIND_PASSWORD"), ) - search_filter = ( - f"(&(objectclass={object_class})(userHomeArea={old_home_area})(!(cn={old_home_area}))(!(endDate=*)))" - ) + search_filter = f"(&(objectclass={object_class})(userHomeArea={old_home_area})(!(cn={old_home_area}))(!(endDate=*)))" ldap_connection.search( ",".join( [ @@ -84,7 +83,9 @@ def change_home_areas( if ldap_connection.result["result"] == 0: log.info(f"Successfully updated {attribute} for {dn}") else: - log.error(f"Failed to update {attribute} for {dn}: {ldap_connection.result}") + log.error( + f"Failed to update {attribute} for {dn}: {ldap_connection.result}" + ) ######################################### @@ -100,13 +101,19 @@ def parse_user_role_list( # and the roles are separated by a semi-colon: # username1,role1;role2;role3|username2,role1;role2 - return {user.split(",")[0]: user.split(",")[1].split(";") for user in user_role_list.split("|")} + return { + user.split(",")[0]: user.split(",")[1].split(";") + for user in user_role_list.split("|") + } def add_roles_to_user(username, roles, user_ou="ou=Users", root_dn="dc=moj,dc=com"): log.info(f"Adding roles {roles} to user {username}") ldap_connection = ldap_connect( - env.vars.get("LDAP_HOST"), env.vars.get("LDAP_USER"), env.secrets.get("LDAP_BIND_PASSWORD") + env.vars.get("LDAP_HOST"), + env.vars.get("LDAP_PORT", 389), + env.vars.get("LDAP_USER"), + env.secrets.get("LDAP_BIND_PASSWORD"), ) for role in roles: try: @@ -131,7 +138,9 @@ def add_roles_to_user(username, roles, user_ou="ou=Users", root_dn="dc=moj,dc=co raise Exception(f"Failed to add role {role} to user {username}") -def process_user_roles_list(user_role_list, user_ou="ou=Users", root_dn="dc=moj,dc=com"): +def process_user_roles_list( + user_role_list, user_ou="ou=Users", root_dn="dc=moj,dc=com" +): user_roles = parse_user_role_list(user_role_list) try: for ( @@ -153,15 +162,28 @@ def process_user_roles_list(user_role_list, user_ou="ou=Users", root_dn="dc=moj, # Update user roles ######################################### - -def update_roles(roles, user_ou, root_dn, add, remove, update_notes, user_note, user_filter, roles_to_filter): +def update_roles( + roles, + user_ou, + root_dn, + add, + remove, + update_notes, + user_note, + user_filter, + roles_to_filter, +): if update_notes and (user_note is None or len(user_note) < 1): log.error("User note must be provided when updating notes") raise Exception("User note must be provided when updating notes") try: - ldap_connection_user_filter = ldap.initialize("ldap://" + env.vars.get("LDAP_HOST")) - ldap_connection_user_filter.simple_bind_s(env.vars.get("LDAP_USER"), env.secrets.get("LDAP_BIND_PASSWORD")) + ldap_connection_user_filter = ldap.initialize( + "ldap://" + env.vars.get("LDAP_HOST") + ) + ldap_connection_user_filter.simple_bind_s( + env.vars.get("LDAP_USER"), env.secrets.get("LDAP_BIND_PASSWORD") + ) except Exception as e: log.exception("Failed to connect to LDAP") raise e @@ -181,7 +203,9 @@ def update_roles(roles, user_ou, root_dn, add, remove, update_notes, user_note, log.exception("Failed to search for users") raise e - users_found = sorted(set([entry[1]["cn"][0].decode("utf-8") for entry in user_filter_results])) + users_found = sorted( + set([entry[1]["cn"][0].decode("utf-8") for entry in user_filter_results]) + ) log.debug("users found from user filter") log.debug(users_found) log.info(f"Found {len(users_found)} users matching the user filter") @@ -198,8 +222,12 @@ def update_roles(roles, user_ou, root_dn, add, remove, update_notes, user_note, log.debug(full_role_filter) try: - ldap_connection_role_filter = ldap.initialize("ldap://" + env.vars.get("LDAP_HOST")) - ldap_connection_role_filter.simple_bind_s(env.vars.get("LDAP_USER"), env.secrets.get("LDAP_BIND_PASSWORD")) + ldap_connection_role_filter = ldap.initialize( + "ldap://" + env.vars.get("LDAP_HOST") + ) + ldap_connection_role_filter.simple_bind_s( + env.vars.get("LDAP_USER"), env.secrets.get("LDAP_BIND_PASSWORD") + ) ldap_connection_role_filter.set_option(ldap.OPT_REFERRALS, 0) except ldap.LDAPError as e: log.exception("Failed to connect to LDAP") @@ -220,14 +248,20 @@ def update_roles(roles, user_ou, root_dn, add, remove, update_notes, user_note, try: response = ldap_connection_role_filter.search_ext( - ",".join([user_ou, root_dn]), ldap.SCOPE_SUBTREE, full_role_filter, ["cn"], serverctrls=[page_control] + ",".join([user_ou, root_dn]), + ldap.SCOPE_SUBTREE, + full_role_filter, + ["cn"], + serverctrls=[page_control], ) while True: pages += 1 log.debug(f"Processing page {pages}") try: - rtype, rdata, rmsgid, serverctrls = ldap_connection_role_filter.result3(response) + rtype, rdata, rmsgid, serverctrls = ldap_connection_role_filter.result3( + response + ) roles_search_result.extend(rdata) cookie = serverctrls[0].cookie print(cookie) @@ -253,7 +287,9 @@ def update_roles(roles, user_ou, root_dn, add, remove, update_notes, user_note, finally: ldap_connection_role_filter.unbind_s() - roles_found = sorted(set({dn.split(",")[1].split("=")[1] for dn, entry in roles_search_result})) + roles_found = sorted( + set({dn.split(",")[1].split("=")[1] for dn, entry in roles_search_result}) + ) roles_found = sorted(roles_found) log.debug("Users found from roles filter: ") @@ -280,6 +316,7 @@ def update_roles(roles, user_ou, root_dn, add, remove, update_notes, user_note, try: ldap_connection_action = ldap_connect( env.vars.get("LDAP_HOST"), + env.vars.get("LDAP_PORT", 389), env.vars.get("LDAP_USER"), env.secrets.get("LDAP_BIND_PASSWORD"), ) @@ -317,7 +354,9 @@ def update_roles(roles, user_ou, root_dn, add, remove, update_notes, user_note, removed = 0 not_removed = 0 failed = 0 - ldap_connection_action.delete(f"cn={item[1]},cn={item[0]},{user_ou},{root_dn}") + ldap_connection_action.delete( + f"cn={item[1]},cn={item[0]},{user_ou},{root_dn}" + ) if ldap_connection_action.result["result"] == 0: log.info(f"Successfully removed role '{item[1]}' from user '{item[0]}'") actioned = actioned + 1 @@ -333,11 +372,15 @@ def update_roles(roles, user_ou, root_dn, add, remove, update_notes, user_note, log.info("\n==========================\n\tSUMMARY\n==========================") log.info("User/role searches:") - log.info(f" - Found {len(roles_found)} users with roles matching the role filter") + log.info( + f" - Found {len(roles_found)} users with roles matching the role filter" + ) log.info(f" - Found {len(users_found)} users matching the user filter") log.info("This produces the following matches:") - log.info(f" - Found {len(matched_users)} users with roles matching the role filter and user filter") + log.info( + f" - Found {len(matched_users)} users with roles matching the role filter and user filter" + ) log.info("Actions:") log.info(f" - Successfully actioned {actioned} roles") @@ -405,11 +448,14 @@ def deactivate_crc_users(user_ou, root_dn): log.info("Deactivating CRC users") ldap_connection = ldap_connect( env.vars.get("LDAP_HOST"), + env.vars.get("LDAP_PORT", 389), env.vars.get("LDAP_USER"), env.secrets.get("LDAP_BIND_PASSWORD"), ) - user_filter = "(userSector=private)(!(userSector=public))(!(endDate=*))(objectclass=NDUser)" + user_filter = ( + "(userSector=private)(!(userSector=public))(!(endDate=*))(objectclass=NDUser)" + ) home_areas = [ [ @@ -479,9 +525,7 @@ def deactivate_crc_users(user_ou, root_dn): connection = cli.database.connection() for user_dn in all_users: try: - update_sql = ( - f"UPDATE USER_ SET END_DATE=TRUNC(CURRENT_DATE) WHERE UPPER(DISTINGUISHED_NAME)=UPPER(:user_dn)" - ) + update_sql = f"UPDATE USER_ SET END_DATE=TRUNC(CURRENT_DATE) WHERE UPPER(DISTINGUISHED_NAME)=UPPER(:user_dn)" update_cursor = connection.cursor() update_cursor.execute( update_sql, @@ -502,6 +546,7 @@ def user_expiry(user_ou, root_dn): ldap_connection_lock = ldap_connect( env.vars.get("LDAP_HOST"), + env.vars.get("LDAP_PORT", 389), env.vars.get("LDAP_USER"), env.secrets.get("LDAP_BIND_PASSWORD"), ) @@ -540,6 +585,7 @@ def user_expiry(user_ou, root_dn): ldap_connection_unlock = ldap_connect( env.vars.get("LDAP_HOST"), + env.vars.get("LDAP_PORT", 389), env.vars.get("LDAP_USER"), env.secrets.get("LDAP_BIND_PASSWORD"), ) @@ -578,6 +624,7 @@ def remove_all_user_passwords(user_ou, root_dn): ldap_connection = ldap_connect( env.vars.get("LDAP_HOST"), + env.vars.get("LDAP_PORT", 389), env.vars.get("LDAP_USER"), env.secrets.get("LDAP_BIND_PASSWORD"), ) @@ -612,7 +659,9 @@ def remove_all_user_passwords(user_ou, root_dn): ] }, ) - log.info(f"Successfully removed passwd for user {user}, or it didn't have one to begin with") + log.info( + f"Successfully removed passwd for user {user}, or it didn't have one to begin with" + ) except Exception as e: log.exception(f"Failed to remove passwd for user {user}") raise e diff --git a/cli/logger.py b/cli/logger.py index 16afdf1..a76bb6a 100644 --- a/cli/logger.py +++ b/cli/logger.py @@ -15,7 +15,9 @@ def __init__( fmt=format_str, datefmt=datefmt_str, ) - self._secrets_set = set(cli.env.secrets.values()) # Retrieve secrets set here + self._secrets_set = set( + cli.env.secrets.values() + ) # Retrieve secrets set here self.default_msec_format = "%s.%03d" def _filter( @@ -23,7 +25,10 @@ def _filter( s, ): redacted = " ".join( - ["*" * len(string) if string in self._secrets_set else string for string in s.split(" ")] + [ + "*" * len(string) if string in self._secrets_set else string + for string in s.split(" ") + ] ) return redacted @@ -37,7 +42,10 @@ def format( print("configure_logging") """Configure logging based on environment variables.""" - format = cli.env.vars.get("LOG_FORMAT") or "%(asctime)s.%(msecs)03d - %(levelname)s: %(message)s" + format = ( + cli.env.vars.get("LOG_FORMAT") + or "%(asctime)s.%(msecs)03d - %(levelname)s: %(message)s" + ) datefmt = cli.env.vars.get("LOG_DATE_FORMAT") or "%Y-%m-%d %H:%M:%S" log = logging.getLogger(__name__) diff --git a/setup.py b/setup.py index d22494e..73fbb7b 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,9 @@ standard_pkgs = [r for r in requirements if not r.startswith("git+")] git_pkgs = [r for r in requirements if r.startswith("git+")] -formatted_git_pkgs = [f"{git_pkg.split('/')[-1].split('.git@')[0]} @ {git_pkg}" for git_pkg in git_pkgs] +formatted_git_pkgs = [ + f"{git_pkg.split('/')[-1].split('.git@')[0]} @ {git_pkg}" for git_pkg in git_pkgs +] all_reqs = standard_pkgs + formatted_git_pkgs setup(