diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..5c8f544 --- /dev/null +++ b/.flake8 @@ -0,0 +1,34 @@ +[flake8] + +# PEP-8 line length is not very practical +max-line-length = 88 + +extend-ignore = + # See https://github.com/PyCQA/pycodestyle/issues/373 + # flake8/pycodechecker give false positives on black code + # line break before ':' apparently gives false positives with black formatter... + E203, + # line break before binary operator, fights with black formatter... + W503, + # importing with '*' ... + F403, + # Bare exception handling, fixing in NIT-854 + E722, + # Missing docstring in public nested class + D104, + # Missing docstring in public package + D103, + # f-string but no variables, e.g. print(f"hello")... + F541, + # Line too long (>79 chars), but should not be firing due to max-line-length = 120 + E501, + # add docustrings + D100, + # to be corrected with NIT-854 + B001 + +# ===================== +# flake-quote settings: +# ===================== +# Set this to match black style: +inline-quotes = double diff --git a/.github/workflows/tag-and-release.yml b/.github/workflows/tag-and-release.yml index 6ba5b4c..6312147 100644 --- a/.github/workflows/tag-and-release.yml +++ b/.github/workflows/tag-and-release.yml @@ -27,32 +27,33 @@ jobs: with: ref: ${{ github.event.pull_request.merge_commit_sha }} fetch-depth: '0' + - name: release or prerelease + id: release_type + run: | + if [[ "${{ github.event.pull_request.base.ref }}" == "main" ]]; then + echo "This is a release" + echo "PRERELEASE=false" >> $GITHUB_OUTPUT + else + echo "This is a prerelease" + echo "PRERELEASE=true" >> $GITHUB_OUTPUT + fi - name: Bump version and push tag id: tag uses: anothrNick/github-tag-action@1.67.0 # Don't use @master or @v1 unless you're happy to test the latest version env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} WITH_V: true - PRE_RELEASE_NAME: dev - - name: release or prerelease - id: release_type - run: | - if [[ ${{ steps.tag.outputs.new_tag }} == *"dev"* ]]; then - echo "This is a prerelease" - echo "DEV=true" >> $GITHUB_OUTPUT - else - echo "This is a release" - echo "DEV=false" >> $GITHUB_OUTPUT - fi + PRERELEASE_SUFFIX: dev + PRERELEASE: "${{ steps.release_type.outputs.PRERELEASE }}" - name: Create prerelease - if: steps.release_type.outputs.DEV == 'true' + if: steps.release_type.outputs.PRERELEASE == 'true' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | gh release create ${{ steps.tag.outputs.new_tag }} --title "Dev ${{ steps.tag.outputs.new_tag }}" --prerelease --generate-notes --verify-tag - name: Create release - if: steps.release_type.outputs.DEV == 'false' + if: steps.release_type.outputs.PRERELEASE == 'false' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - gh release create ${{ steps.tag.outputs.new_tag }} --title "Release ${{ steps.tag.outputs.new_tag }}" --generate-notes --verify-tag \ No newline at end of file + gh release create ${{ steps.tag.outputs.new_tag }} --title "Release ${{ steps.tag.outputs.new_tag }}" --generate-notes --verify-tag diff --git a/.gitignore b/.gitignore index 6887167..d2bd538 100644 --- a/.gitignore +++ b/.gitignore @@ -159,4 +159,12 @@ cython_debug/ # option (not recommended) you can uncomment the following to ignore the entire idea folder. .idea/ # VSCode Config -.vscode \ No newline at end of file +.vscode + +.secrets +.vars +/rbac +/rendered + +*.ldif +*.ldif.j2 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..1e1b806 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,43 @@ +# pre-commit run --all-files +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: check-added-large-files + - id: check-case-conflict + - id: check-executables-have-shebangs + - id: check-merge-conflict + - id: check-shebang-scripts-are-executable + - id: check-symlinks + - id: check-yaml + - id: debug-statements + exclude: tests/ + - id: destroyed-symlinks + - id: end-of-file-fixer + exclude: tests/test_changes/ + files: \.(py|sh|rst|yml|yaml)$ + - id: mixed-line-ending + - id: trailing-whitespace + files: \.(py|sh|rst|yml|yaml)$ + - repo: https://github.com/psf/black + rev: 23.9.1 + hooks: + - id: black + - repo: https://github.com/PyCQA/flake8 + rev: 6.1.0 + hooks: + - id: flake8 + additional_dependencies: [ + 'flake8-blind-except', + 'flake8-docstrings', + 'flake8-bugbear', + 'flake8-comprehensions', + 'flake8-docstrings', + 'flake8-implicit-str-concat', + 'pydocstyle>=5.0.0', + ] + - repo: https://github.com/codespell-project/codespell + rev: v2.2.5 + hooks: + - id: codespell + files: \.(py|sh|rst|yml|yaml)$ diff --git a/cli/__init__.py b/cli/__init__.py index fca6d0f..f370ca6 100644 --- a/cli/__init__.py +++ b/cli/__init__.py @@ -1,28 +1,206 @@ import click -from cli import ldap +import cli.ldap_cmds.rbac +import cli.ldap_cmds.user + +from cli import ( + logger, +) -from cli import git @click.group() def main_group(): pass + @click.command() -@click.option("--user-ou", help="OU to add users to, defaults to ou=Users", default="ou=Users") -@click.option("--root-dn", help="Root DN to add users to", default="dc=moj,dc=com") -@click.argument("user-role-list", required=True) -def add_roles_to_users(user_ou, root_dn, user_role_list): - ldap.process_user_roles_list(user_role_list, user_ou, root_dn) +@click.option( + "-u", + "--user-ou", + help="OU to add users to, defaults to ou=Users", + default="ou=Users", +) +@click.option( + "-r", + "--root-dn", + help="Root DN to add users to", + default="dc=moj,dc=com", +) +@click.argument( + "user-role-list", + required=True, +) +def add_roles_to_users( + user_ou, + root_dn, + user_role_list, +): + cli.ldap.user.process_user_roles_list( + user_role_list, + user_ou, + root_dn, + ) +# Update user home area @click.command() -def git_test(): - git.dl_test() +@click.option( + "-o", + "--old-home-area", + help="name of old home area", + required=True, +) +@click.option( + "-n", + "--new-home-area", + help="name of new home area", + required=True, +) +@click.option( + "-u", + "--user-ou", + help="OU to add users to, defaults to ou=Users", + default="ou=Users", +) +@click.option( + "-r", + "--root-dn", + help="Root DN to add users to, defaults to dc=moj,dc=com", + default="dc=moj,dc=com", +) +def update_user_home_areas( + old_home_area, + new_home_area, + user_ou, + root_dn, +): + cli.ldap.user.change_home_areas( + old_home_area, + new_home_area, + user_ou, + root_dn, + ) + + +# Update user roles +@click.command() +@click.argument( + "roles", + required=True, +) +@click.argument( + "user-note", + required=False, +) +@click.option( + "-u", + "--user-ou", + help="OU to add users to, defaults to ou=Users", + default="ou=Users", +) +@click.option( + "-r", + "--root-dn", + help="Root DN to add users to, defaults to dc=moj,dc=com", + default="dc=moj,dc=com", +) +@click.option( + "--add", + help="Add role to users", + is_flag=True, +) +@click.option( + "--remove", + help="Remove role from users", + is_flag=True, +) +@click.option( + "--update-notes", + help="Remove role from users", + is_flag=True, +) +@click.option( + "-rf", + "--role-filter", + help='Comma separated string to generate roles filter from eg "role1,role2,role3"', + required=False, + default="*", +) +@click.option( + "-uf", + "--user-filter", + help="Filter to find users", + required=False, + default="(userSector=*)", +) +def update_user_roles( + roles, + user_ou, + root_dn, + add, + remove, + update_notes, + user_note, + user_filter, + role_filter, +): + cli.ldap.user.update_roles( + roles, + user_ou, + root_dn, + add, + remove, + update_notes, + user_note=user_note, + user_filter=user_filter, + role_filter=role_filter, + ) + + +@click.command() +@click.option( + "-t", + "--rbac-repo-tag", + help="RBAC repo tag to use", + default="master", +) +def rbac_uplift( + rbac_repo_tag, +): + cli.ldap_cmds.rbac.main(rbac_repo_tag) + + +@click.command() +@click.option( + "-u", + "--user-ou", + help="OU to add users to, defaults to ou=Users", + default="ou=Users", +) +@click.option( + "-r", + "--root-dn", + help="Root DN to add users to, defaults to dc=moj,dc=com", + default="dc=moj,dc=com", +) +def deactivate_crc_users( + user_ou, + root_dn, +): + cli.ldap.user.deactivate_crc_users( + user_ou, + root_dn, + ) + # from cli.ldap import test main_group.add_command(add_roles_to_users) -main_group.add_command(git_test) +main_group.add_command(rbac_uplift) +main_group.add_command(update_user_home_areas) +main_group.add_command(update_user_roles) +main_group.add_command(deactivate_crc_users) + +logger.configure_logging() if __name__ == "__main__": main_group() diff --git a/cli/ansible/__init__.py b/cli/ansible/__init__.py deleted file mode 100644 index 98c7787..0000000 --- a/cli/ansible/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -import ansible_runner - diff --git a/cli/config.py b/cli/config.py deleted file mode 100644 index 5a7343a..0000000 --- a/cli/config.py +++ /dev/null @@ -1,17 +0,0 @@ -import os - -from dotenv import load_dotenv - -load_dotenv() - -ldap_host = os.getenv("LDAP_HOST") -ldap_user = os.getenv("LDAP_USER") -ldap_password = os.getenv("LDAP_PASSWORD") -db_user = os.getenv("DB_USER") -db_password = os.getenv("DB_PASSWORD") -db_host = os.getenv("DB_HOST") -db_port = os.getenv("DB_PORT") -db_service_name = os.getenv("DB_SERVICE_NAME") -gh_app_id = os.getenv("GH_APP_ID") -gh_private_key = os.getenv("GH_PRIVATE_KEY") -gh_installation_id = os.getenv("GH_INSTALLATION_ID") \ No newline at end of file diff --git a/cli/database/__init__.py b/cli/database/__init__.py new file mode 100644 index 0000000..511366b --- /dev/null +++ b/cli/database/__init__.py @@ -0,0 +1,19 @@ +import oracledb +from cli import ( + env, +) +from cli.logger import ( + log, +) + + +def connection(): + try: + conn = oracledb.connect(env.secrets.get("DB_CONNECTION_STRING")) + log.debug("Created database connection successfully") + return conn + except Exception as e: + log.exception( + f"Failed to create database connection. An exception of type {type(e).__name__} occurred: {e}" + ) + raise e diff --git a/cli/env.py b/cli/env.py new file mode 100644 index 0000000..98c815f --- /dev/null +++ b/cli/env.py @@ -0,0 +1,88 @@ +import os + +from dotenv import ( + dotenv_values, +) + +import ast + +# ldap_host = os.getenv("LDAP_HOST") +# ldap_user = os.getenv("LDAP_USER") +# ldap_password = os.getenv("LDAP_PASSWORD") +# db_user = os.getenv("DB_USER") +# db_password = os.getenv("DB_PASSWORD") +# db_host = os.getenv("DB_HOST") +# db_port = os.getenv("DB_PORT") +# db_service_name = os.getenv("DB_SERVICE_NAME") +# gh_app_id = os.getenv("GH_APP_ID") +# gh_private_key = os.getenv("GH_PRIVATE_KEY") +# gh_installation_id = os.getenv("GH_INSTALLATION_ID") + + +vars = { + **{ + key.replace( + "VAR_", + "", + ).replace( + "_DICT", + "", + ): ast.literal_eval(val) + if "DICT" in key + else val + for key, val in dotenv_values(".vars").items() + if val is not None + }, # load development variables + **{ + key.replace( + "VAR_", + "", + ).replace( + "_DICT", + "", + ): ast.literal_eval(val) + if "DICT" in key + else val + for key, val in os.environ.items() + if key.startswith("VAR_") and val is not None + }, +} +# loads all environment variables starting with SECRET_ into a dictionary +secrets = { + **{ + key.replace( + "SECRET_", + "", + ) + .replace( + "_DICT", + "", + ) + .replace( + "SSM_", + "", + ): ast.literal_eval(val) + if "_DICT" in key + else val + for key, val in dotenv_values(".secrets").items() + if val is not None + }, + **{ + key.replace( + "SECRET_", + "", + ) + .replace( + "_DICT", + "", + ) + .replace( + "SSM_", + "", + ): ast.literal_eval(val) + if "DICT" in key + else val + for key, val in os.environ.items() + if key.startswith("SECRET_") or key.startswith("SSM_") and val is not None + }, +} diff --git a/cli/git/__init__.py b/cli/git/__init__.py index ad06a28..8a733c9 100644 --- a/cli/git/__init__.py +++ b/cli/git/__init__.py @@ -1,50 +1,103 @@ -from github import Github, Auth -from git import Repo +from git import ( + Repo, +) import jwt import time import requests import logging -from cli import config -def get_access_token(app_id, private_key, installation_id): + + +def get_access_token( + app_id, + private_key, + installation_id, +): # Create a JSON Web Token (JWT) using the app's private key now = int(time.time()) payload = { "iat": now, "exp": now + 600, - "iss": app_id + "iss": app_id, } - jwt_token = jwt.encode(payload, private_key, algorithm="RS256") + jwt_token = jwt.encode( + payload, + private_key, + algorithm="RS256", + ) # Exchange the JWT for an installation access token headers = { "Authorization": f"Bearer {jwt_token}", - "Accept": "application/vnd.github.v3+json" + "Accept": "application/vnd.github.v3+json", } - response = requests.post(f"https://api.github.com/app/installations/{installation_id}/access_tokens", - headers=headers) + try: + response = requests.post( + f"https://api.github.com/app/installations/{installation_id}/access_tokens", + headers=headers, + ) + except Exception as e: + logging.exception( + f"Failed to get access token. An exception of type {type(e).__name__} occurred: {e}" + ) + raise e + # extract the token from the response access_token = response.json().get("token") return access_token -def get_repo(url, token=None, auth_type="x-access-token", dest_name="repo"): + +def get_repo( + url, + depth="1", + branch_or_tag="master", + token=None, + auth_type="x-access-token", + dest_name="repo", +): # if there is an @ in the url, assume auth is already specified - if '@' in url: - logging.info('auth already specified in url') - return Repo.clone_from(url, dest_name) + multi_options = [ + "--depth " + depth, + "--branch " + branch_or_tag, + ] + if "@" in url: + logging.info("auth already specified in url") + try: + return Repo.clone_from( + url, + dest_name, + multi_options=multi_options, + ) + except Exception as e: + logging.exception( + f"Failed to clone repo. An exception of type {type(e).__name__} occurred: {e}" + ) + raise e # if there is a token, assume auth is required and use the token and auth_type elif token: templated_url = f'https://{auth_type}:{token}@{url.split("//")[1]}' - logging.info(f'cloning with token: {templated_url}') - return Repo.clone_from(templated_url, dest_name) + logging.info(f"cloning with token: {templated_url}") + try: + return Repo.clone_from( + templated_url, + dest_name, + multi_options=multi_options, + ) + except Exception as e: + logging.exception( + f"Failed to clone repo. An exception of type {type(e).__name__} occurred: {e}" + ) + raise e # if there is no token, assume auth is not required and clone without else: - logging.info('cloning without auth') - return Repo.clone_from(url, dest_name) -def dl_test(): - app_id = config.gh_app_id - private_key = config.gh_private_key - installation_id = config.gh_installation_id - url = 'https://github.com/ministryofjustice/hmpps-delius-pipelines.git' - token = get_access_token(app_id, private_key, installation_id) - repo = get_repo(url, token=token, dest_name='delius-pipelines') - print(repo) \ No newline at end of file + logging.info("cloning without auth") + try: + return Repo.clone_from( + url, + dest_name, + multi_options=multi_options, + ) + except Exception as e: + logging.exception( + f"Failed to clone repo. An exception of type {type(e).__name__} occurred: {e}" + ) + raise e diff --git a/cli/ldap/add_roles_to_username.py b/cli/ldap/add_roles_to_username.py deleted file mode 100644 index 0ae4396..0000000 --- a/cli/ldap/add_roles_to_username.py +++ /dev/null @@ -1,35 +0,0 @@ -import logging - -from cli import config -from cli.ldap import ldap_connect - - -def parse_user_role_list(user_role_list): - return {user.split(",")[0]: user.split(",")[1].split(";") for user in user_role_list.split("|")} - - -def add_roles_to_user(username, roles, user_ou="ou=Users", root_dn="dc=moj,dc=com"): - logging.info(f"Adding roles {roles} to user {username}") - ldap_connection = ldap_connect(config.ldap_host, config.ldap_user, config.ldap_password) - for role in roles: - ldap_connection.add( - f"cn={role},cn={username},{user_ou},{root_dn}", - attributes={ - "objectClass": ["NDRoleAssociation", "alias"], - "aliasedObjectName": f"cn={role},cn={username},cn=ndRoleCatalogue,{root_dn}", - }, - ) - if ldap_connection.result["result"] == 0: - print(f"Successfully added role {role} to user {username}") - elif ldap_connection.result["result"] == 68: - print(f"Role {role} already exists for user {username}") - else: - print(ldap_connection.result) - print(ldap_connection.response) - raise Exception(f"Failed to add role {role} to user {username}") - - -def process_user_roles_list(user_role_list, user_ou="ou=Users", root_dn="dc=moj,dc=com"): - user_roles = parse_user_role_list(user_role_list) - for user, roles in user_roles.items(): - add_roles_to_user(user, roles, user_ou, root_dn) diff --git a/cli/ldap/test.py b/cli/ldap/test.py deleted file mode 100644 index 1070de6..0000000 --- a/cli/ldap/test.py +++ /dev/null @@ -1,15 +0,0 @@ -from cli import config -from cli.ldap import ldap_connect -from ldap3 import LEVEL -import click - - -def test_search(): - ldap_connection = ldap_connect(config.ldap_host, config.ldap_user, config.ldap_password) - ldap_connection.search("dc=moj,dc=com", "(objectClass=*)", search_scope=LEVEL, attributes=["*"], time_limit=120) - print(ldap_connection.entries) - - -@click.command() -def test(): - test_search() diff --git a/cli/ldap/__init__.py b/cli/ldap_cmds/__init__.py similarity index 59% rename from cli/ldap/__init__.py rename to cli/ldap_cmds/__init__.py index 126deb7..69110ed 100644 --- a/cli/ldap/__init__.py +++ b/cli/ldap_cmds/__init__.py @@ -1,14 +1,23 @@ -from ldap3 import Server, Connection, ALL +from ldap3 import ( + Server, + Connection, +) -# import oracledb -import logging - -logging.basicConfig(level=logging.DEBUG) +# import oracledb +def ldap_connect( + ldap_host, + ldap_user, + ldap_password, +): + server = Server(ldap_host) -def ldap_connect(ldap_host, ldap_user, ldap_password): return Connection( - server=ldap_host, user=ldap_user, password=ldap_password, auto_bind="NO_TLS", authentication="SIMPLE" + server=server, + user=ldap_user, + password=ldap_password, + auto_bind="NO_TLS", + authentication="SIMPLE", ) diff --git a/cli/ldap_cmds/rbac.py b/cli/ldap_cmds/rbac.py new file mode 100644 index 0000000..33a30b4 --- /dev/null +++ b/cli/ldap_cmds/rbac.py @@ -0,0 +1,518 @@ +from pprint import pprint + +import ldap +import ldap3.utils.hashed +import ldif +import ldap.modlist as modlist + +from cli.ldap_cmds import ( + ldap_connect, +) +from cli import ( + env, +) +import cli.git as git +import glob +from cli.logger import ( + log, +) +from pathlib import ( + Path, +) +import cli.template + +# example for token auth +# def get_repo_with_token(repo_tag="master"): +# app_id = env.vars.get("GH_APP_ID") +# private_key = env.vars.get("GH_PRIVATE_KEY") +# installation_id = env.vars.get("GH_INSTALLATION_ID") +# token = git.get_access_token(app_id, private_key, installation_id) + + +ldap_config = { + "bind_user": "cn=root,dc=moj,dc=com", + "bind_user_cn": "root", + "base_root": "dc=moj,dc=com", + "base_root_dc": "moj", + "base_users": "ou=Users,dc=moj,dc=com", + "base_users_ou": "Users", + "base_service_users": "cn=EISUsers,ou=Users,dc=moj,dc=com", + "base_roles": "cn=ndRoleCatalogue,ou=Users,dc=moj,dc=com", + "base_role_groups": "cn=ndRoleGroups,ou=Users,dc=moj,dc=com", + "base_groups": "ou=groups,dc=moj,dc=com", + "base_groups_ou": "groups", +} + + +def get_repo( + repo_tag="master", +): + url = "https://github.com/ministryofjustice/hmpps-ndelius-rbac.git" + try: + repo = git.get_repo( + url, + dest_name="rbac", + branch_or_tag=repo_tag, + ) + return repo + except Exception as e: + log.exception(e) + return None + + +def prep_for_templating( + files, + strings=None, +): + rbac_substitutions = { + "bind_password_hash.stdout": "bind_password_hash", + r"ldap_config.base_users | regex_replace('^.+?=(.+?),.*$', '\\1')": "ldap_config.base_users_ou", + r"ldap_config.base_root | regex_replace('^.+?=(.+?),.*$', '\\1')": "ldap_config.base_root_dc", + r"ldap_config.base_groups | regex_replace('^.+?=(.+?),.*$', '\\1')": "ldap_config.base_groups_ou", + r"ldap_config.bind_user | regex_replace('^.+?=(.+?),.*$', '\\1')": "ldap_config.bind_user_cn", + "'/'+environment_name+'/'+project_name+'": "", + "/gdpr/api/": "'gdpr_api_", + "/pwm/pwm/config_password": "'pwm_config_password", + "/merge/api/client_secret": "'merge_api_client_secret", + "/weblogic/ndelius-domain/umt_client_secret": "'umt_client_secret", + "ssm_prefix + ": "", + "cn=Users,dc=pcms,dc=internal": "ou=Users,dc=moj,dc=com", + "ssm_prefix+": "", + } + + if strings is None: + strings = env.vars.get("RBAC_SUBSTITUTIONS") or rbac_substitutions + + for file_path in files: + file = Path(file_path) + log.info("Replacing strings in rbac files") + for ( + k, + v, + ) in strings.items(): + log.debug(f"replacing {k} with {v} in {file_path}") + file.write_text( + file.read_text().replace( + k, + v, + ), + ) + + +def template_rbac( + files, +): + hashed_pwd_admin_user = ldap3.utils.hashed.hashed( + ldap3.HASHED_SALTED_SHA, + env.secrets.get("LDAP_ADMIN_PASSWORD"), + ) + rendered_files = [] + + for file in files: + rendered_text = cli.template.render( + file, + ldap_config=env.vars.get("LDAP_CONFIG") or ldap_config, + bind_password_hash=hashed_pwd_admin_user, + secrets=env.secrets, + oasys_password=env.secrets.get("OASYS_PASSWORD"), + environment_name=env.vars.get("ENVIRONMENT_NAME"), + project_name=env.vars.get("PROJECT_NAME"), + ) + rendered_file = cli.template.save( + rendered_text, + file, + ) + rendered_files.append(rendered_file) + return rendered_files + + +def context_ldif( + rendered_files, +): + context_file = [file for file in rendered_files if "context" in Path(file).name] + + # connect to ldap + try: + connection = ldap.initialize("ldap://" + env.vars.get("LDAP_HOST")) + connection.simple_bind_s(env.vars.get("LDAP_USER"), env.secrets.get("LDAP_BIND_PASSWORD")) + except Exception as e: + log.exception(f"Failed to connect to ldap") + raise e + + for file in context_file: + # parse the ldif into dn and record + + records = ldif.LDIFRecordList(open(file, "rb")) + records.parse() + + pprint(records.all_records) + # loop through the records + for entry in records.all_records: + dn = entry[0] + attributes = entry[1] + log.info(f"got entry record: {dn}") + log.debug(attributes) + + try: + connection.add_s( + dn, + modlist.addModlist(attributes), + ) + except ldap.ALREADY_EXISTS as already_exists_e: + log.info(f"{dn} already exists") + log.debug(already_exists_e) + except Exception as e: + log.exception(f"Failed to add {dn}... {attributes}") + raise e + + +def group_ldifs( + rendered_files, +): + # connect to ldap + try: + connection = ldap.initialize("ldap://" + env.vars.get("LDAP_HOST")) + connection.simple_bind_s(env.vars.get("LDAP_USER"), env.secrets.get("LDAP_BIND_PASSWORD")) + except Exception as e: + log.exception(f"Failed to connect to ldap") + raise e + + group_files = [file for file in rendered_files if "-groups" in Path(file).name] + # loop through the group files + for file in group_files: + # parse the ldif into dn and record + + records = ldif.LDIFRecordList(open(file, "rb")) + records.parse() + + pprint(records.all_records) + # loop through the records + for entry in records.all_records: + dn = entry[0] + attributes = entry[1] + log.debug(f"got entry record: {dn}") + log.debug(attributes) + # add the record to ldap + try: + connection.add_s( + dn, + modlist.addModlist(attributes), + ) + except ldap.ALREADY_EXISTS as already_exists_e: + log.info(f"{dn} already exists") + log.debug(already_exists_e) + except Exception as e: + log.exception(f"Failed to add {dn}... {attributes}") + raise e + + if attributes.get("description"): + log.info(f"Updating description for {dn}") + try: + connection.modify(dn, [(ldap.MOD_REPLACE, "description", attributes["description"])]) + except ldap.ALREADY_EXISTS as already_exists_e: + log.info(f"{dn} already exists") + log.debug(already_exists_e) + except Exception as e: + log.exception(f"Failed to add {dn}... {attributes}") + raise e + + +def policy_ldifs( + rendered_files, +): + # connect to ldap + try: + connection = ldap.initialize("ldap://" + env.vars.get("LDAP_HOST")) + connection.simple_bind_s(env.vars.get("LDAP_USER"), env.secrets.get("LDAP_BIND_PASSWORD")) + except Exception as e: + log.exception(f"Failed to connect to ldap") + raise e + + policy_files = [file for file in rendered_files if "policy" in Path(file).name] + + # first, delete the policies + ldap_config_dict = env.vars.get("LDAP_CONFIG") or ldap_config + policy_tree = "ou=Policies," + ldap_config_dict.get("base_root") + + tree = connection.search_s( + policy_tree, + ldap.SCOPE_SUBTREE, + "(objectClass=*)", + ) + tree.reverse() + + for entry in tree: + try: + log.debug(entry[0]) + connection.delete_ext_s(entry[0], serverctrls=[ldap.controls.simple.ManageDSAITControl()]) + print(f"Deleted {entry[0]}") + except ldap.NO_SUCH_OBJECT as no_such_object_e: + log.info("No such object found, 32") + log.debug(no_such_object_e) + + # loop through the policy files + for file in policy_files: + # parse the ldif into dn and record + + records = ldif.LDIFRecordList(open(file, "rb")) + records.parse() + + pprint(records.all_records) + # loop through the records + for entry in records.all_records: + dn = entry[0] + attributes = entry[1] + log.info(f"Got entry record: {dn}") + # add the record to ldap + try: + connection.add_s( + dn, + modlist.addModlist(attributes), + ) + except ldap.ALREADY_EXISTS as already_exists_e: + log.info(f"{dn} already exists") + log.debug(already_exists_e) + except Exception as e: + log.exception(f"Failed to add {dn}... {attributes}") + raise e + + +def role_ldifs( + rendered_files, +): + # connect to ldap + try: + connection = ldap.initialize("ldap://" + env.vars.get("LDAP_HOST")) + connection.simple_bind_s(env.vars.get("LDAP_USER"), env.secrets.get("LDAP_BIND_PASSWORD")) + except Exception as e: + log.exception(f"Failed to connect to ldap") + raise e + + role_files = [file for file in rendered_files if "nd_role" in Path(file).name] + + # first, delete the roles + ldap_config_dict = env.vars.get("LDAP_CONFIG") or ldap_config + + role_trees = [ + "cn=ndRoleCatalogue," + ldap_config_dict.get("base_users"), + "cn=ndRoleGroups," + ldap_config_dict.get("base_users"), + ] + for role_tree in role_trees: + tree = connection.search_s( + role_tree, + ldap.SCOPE_SUBTREE, + "(objectClass=*)", + ) + tree.reverse() + + for entry in tree: + try: + log.debug(entry[0]) + connection.delete_ext_s(entry[0], serverctrls=[ldap.controls.simple.ManageDSAITControl()]) + print(f"Deleted {entry[0]}") + except ldap.NO_SUCH_OBJECT as no_such_object_e: + log.info("No such object found, 32") + log.debug(no_such_object_e) + + # ensure boolean values are Uppercase.. this comes from the ansible yml + # (not yet implemented, probably not needed) + + # loop through the role files + for file in role_files: + # parse the ldif into dn and record + + records = ldif.LDIFRecordList(open(file, "rb")) + records.parse() + + pprint(records.all_records) + # loop through the records + for entry in records.all_records: + dn = entry[0] + attributes = entry[1] + log.info(f"Got entry record: {dn}") + # add the record to ldap + try: + connection.add_s( + dn, + modlist.addModlist(attributes), + ) + except ldap.ALREADY_EXISTS as already_exists_e: + log.info(f"{dn} already exists") + log.debug(already_exists_e) + except Exception as e: + log.exception(f"Failed to add {dn}... {attributes}") + raise e + + +# not complete!! +# see https://github.com/ministryofjustice/hmpps-delius-pipelines/blob/master/components/delius-core/playbooks/rbac/import_schemas.yml +def schema_ldifs( + rendered_files, +): + # connect to ldap + try: + connection = ldap.initialize("ldap://" + env.vars.get("LDAP_HOST")) + connection.simple_bind_s(env.vars.get("LDAP_USER"), env.secrets.get("LDAP_BIND_PASSWORD")) + except Exception as e: + log.exception(f"Failed to connect to ldap") + raise e + + schema_files = [file for file in rendered_files if "delius.ldif" or "pwm.ldif" in Path(file).name] + + # loop through the schema files + for file in schema_files: + # parse the ldif into dn and record + records = ldif.LDIFRecordList(open(file, "rb")) + records.parse() + # loop through the records + for entry in records.all_records: + log.info(f"Got entry record: {dn}") + # add the record to ldap + try: + dn = entry[0] + attributes = entry[1] + print(f" {entry[0]}") + connection.add_s(dn, modlist.addModlist(attributes)) + except ldap.ALREADY_EXISTS as already_exists_e: + log.info(f"{dn} already exists") + log.debug(already_exists_e) + except Exception as e: + log.exception(f"Failed to add {dn}... {attributes}") + raise e + + +def user_ldifs( + rendered_files, +): + # connect to ldap + try: + connection = ldap.initialize("ldap://" + env.vars.get("LDAP_HOST")) + connection.simple_bind_s(env.vars.get("LDAP_USER"), env.secrets.get("LDAP_BIND_PASSWORD")) + except Exception as e: + log.exception(f"Failed to connect to ldap") + raise e + + except Exception as e: + log.exception(f"Failed to connect to ldap") + raise e + + user_files = [file for file in rendered_files if "-users.ldif" in Path(file).name] + + # first, delete the users + for file in user_files: + records = ldif.LDIFRecordList(open(file, "rb")) + records.parse() + + for record in records.all_records: + dn = record[0] + log.info(f"Got entry record: {dn}") + try: + # search for dn children + tree = connection.search_s( + dn, + ldap.SCOPE_SUBTREE, + "(objectClass=*)", + ) + tree.reverse() + print(tree) + for entry in tree: + try: + log.debug(entry[0]) + connection.delete_ext_s(entry[0], serverctrls=[ldap.controls.simple.ManageDSAITControl()]) + print(f"Deleted {entry[0]}") + except ldap.NO_SUCH_OBJECT as no_such_object_e: + log.info("No such object found, 32") + log.debug(no_such_object_e) + # connection.delete_ext_s(dn, serverctrls=[ldap.controls.simple.ManageDSAITControl()]) + # print(f"Deleted {dn}") + except ldap.NO_SUCH_OBJECT as no_such_object_e: + log.info("No such object found, 32") + log.debug(no_such_object_e) + except Exception as e: + log.exception(e) + raise e + + for file in user_files: + records = ldif.LDIFRecordList(open(file, "rb")) + records.parse() + + pprint(records.all_records) + # loop through the records + for entry in records.all_records: + dn = entry[0] + attributes = entry[1] + print(f" {entry[0]}") + connection.add_s(dn, modlist.addModlist(attributes)) + + # connect to ldap + # try: + # ldap_connection_addition = ldap_connect( + # env.vars.get("LDAP_HOST"), + # env.vars.get("LDAP_USER"), + # env.secrets.get("LDAP_BIND_PASSWORD"), + # ) + # except Exception as e: + # log.exception(f"Failed to connect to ldap") + # raise e + + # loop through the user files + # for file in user_files: + # # parse the ldif into dn and record + # parser = LDIFParser( + # open( + # file, + # "rb", + # ), + # strict=False, + # ) + # # loop through the records + # for ( + # dn, + # record, + # ) in parser.parse(): + # log.info(f"Got entry record: {dn}") + # + # # add the record to ldap + # try: + # print(dn) + # print(record) + # ldap_connection_addition.add( + # dn, + # record, + # ) + # except Exception as e: + # log.exception(f"Failed to add {dn}... {record}") + # raise e + # + # if ldap_connection_addition.result["result"] == 0: + # log.info(f"Successfully added users") + # elif ldap_connection_addition.result["result"] == 68: + # log.info(f"{dn} already exists") + # else: + # log.debug(ldap_connection_addition.result) + # log.debug(ldap_connection_addition.response) + # raise Exception(f"Failed to add {dn}... {record}") + + +def main( + rbac_repo_tag, + clone_path="./rbac", +): + get_repo(rbac_repo_tag) + files = [ + file + for file in glob.glob( + f"{clone_path}/**/*", + recursive=True, + ) + if Path(file).is_file() and Path(file).name.endswith(".ldif") or Path(file).name.endswith(".j2") + ] + + prep_for_templating(files) + rendered_files = template_rbac(files) + context_ldif(rendered_files) + policy_ldifs(rendered_files) + # schema_ldifs(files) probably not needed, but need to check! + role_ldifs(rendered_files) + group_ldifs(rendered_files) + user_ldifs(rendered_files) diff --git a/cli/ldap_cmds/user.py b/cli/ldap_cmds/user.py new file mode 100644 index 0000000..410a801 --- /dev/null +++ b/cli/ldap_cmds/user.py @@ -0,0 +1,480 @@ +import oracledb + +import cli.ldap_cmds + +from cli.logger import ( + log, +) +from cli import ( + env, +) + +from cli.ldap_cmds import ( + ldap_connect, +) +from ldap3 import ( + MODIFY_REPLACE, + DEREF_NEVER, +) + +import cli.database +from itertools import ( + product, +) + +from datetime import ( + datetime, +) + + +######################################### +# Change a users home area +######################################### +def change_home_areas( + old_home_area, + new_home_area, + user_ou, + root_dn, + attribute="userHomeArea", + object_class="NDUser", +): + log.info(f"Updating user home areas from {old_home_area} to {new_home_area}") + ldap_connection = ldap_connect( + env.vars.get("LDAP_HOST"), + env.vars.get("LDAP_USER"), + env.secrets.get("LDAP_BIND_PASSWORD"), + ) + + search_filter = ( + f"(&(objectclass={object_class})(userHomeArea={old_home_area})(!(cn={old_home_area}))(!(endDate=*)))" + ) + ldap_connection.search( + ",".join( + [ + user_ou, + root_dn, + ] + ), + search_filter, + attributes=[attribute], + ) + + # Iterate through the search results and update the attribute + for entry in ldap_connection.entries: + dn = entry.entry_dn + changes = { + attribute: [ + ( + MODIFY_REPLACE, + [new_home_area], + ) + ] + } + ldap_connection.modify( + dn, + changes, + ) + + # Check if the modification was successful + if ldap_connection.result["result"] == 0: + log.info(f"Successfully updated {attribute} for {dn}") + else: + log.error(f"Failed to update {attribute} for {dn}: {ldap_connection.result}") + + +######################################### +# Add roles to a user +######################################### + + +def parse_user_role_list( + user_role_list, +): + # The format of the list should be a pipe separated list of username and role lists, + # where the username and role list is separated by a comma character, + # and the roles are separated by a semi-colon: + # username1,role1;role2;role3|username2,role1;role2 + + return {user.split(",")[0]: user.split(",")[1].split(";") for user in user_role_list.split("|")} + + +def add_roles_to_user( + username, + roles, + user_ou="ou=Users", + root_dn="dc=moj,dc=com", +): + log.info(f"Adding roles {roles} to user {username}") + ldap_connection = ldap_connect( + env.vars.get("LDAP_HOST"), + env.vars.get("LDAP_USER"), + env.secrets.get("LDAP_BIND_PASSWORD"), + ) + for role in roles: + try: + ldap_connection.add( + f"cn={role},cn={username},{user_ou},{root_dn}", + attributes={ + "objectClass": [ + "NDRoleAssociation", + "alias", + ], + "aliasedObjectName": f"cn={role},cn={username},cn=ndRoleCatalogue,{user_ou},{root_dn}", + }, + ) + except Exception as e: + log.exception(f"Failed to add role {role} to user {username}") + raise e + + if ldap_connection.result["result"] == 0: + log.info(f"Successfully added role {role} to user {username}") + elif ldap_connection.result["result"] == 68: + log.info(f"Role {role} already exists for user {username}") + else: + log.debug(ldap_connection.result) + log.debug(ldap_connection.response) + raise Exception(f"Failed to add role {role} to user {username}") + + +def process_user_roles_list( + user_role_list, + user_ou="ou=Users", + root_dn="dc=moj,dc=com", +): + log.info(f"secrets: {env.secrets}") + user_roles = parse_user_role_list(user_role_list) + try: + for ( + user, + roles, + ) in user_roles.items(): + add_roles_to_user( + user, + roles, + user_ou, + root_dn, + ) + except Exception as e: + log.exception(f"Failed to add role to user") + raise e + + +######################################### +# Update user roles +######################################### + + +def update_roles( + roles, + user_ou, + root_dn, + add, + remove, + update_notes, + user_note, + user_filter="(userSector=*)", + role_filter="*", +): + if update_notes and (user_note is None or len(user_note) < 1): + log.error("User note must be provided when updating notes") + raise Exception("User note must be provided when updating notes") + + try: + ldap_connection_user_filter = ldap_connect( + env.vars.get("LDAP_HOST"), + env.vars.get("LDAP_USER"), + env.secrets.get("LDAP_BIND_PASSWORD"), + ) + except Exception as e: + log.exception("Failed to connect to LDAP") + raise e + + # # Search for users matching the user_filter + try: + ldap_connection_user_filter.search( + ",".join( + [ + user_ou, + root_dn, + ] + ), + user_filter, + attributes=["cn"], + ) + except Exception as e: + log.exception("Failed to search for users") + raise e + + users_found = sorted([entry.cn.value for entry in ldap_connection_user_filter.entries if entry.cn.value]) + log.debug("users found from user filter") + log.debug(users_found) + ldap_connection_user_filter.unbind() + + roles_filter_list = role_filter.split(",") + roles = roles.split(",") + + # create role filter + if len(roles_filter_list) > 0: + full_role_filter = ( + f"(&(objectclass=NDRoleAssociation)(|{''.join(['(cn=' + role + ')' for role in roles_filter_list])}))" + ) + else: + full_role_filter = "(&(objectclass=NDRoleAssociation)(cn=*))" + + # Search for roles matching the role_filter + + try: + ldap_connection_role_filter = ldap_connect( + env.vars.get("LDAP_HOST"), + env.vars.get("LDAP_USER"), + env.secrets.get("LDAP_BIND_PASSWORD"), + ) + except Exception as e: + log.exception("Failed to connect to LDAP") + raise e + + try: + ldap_connection_role_filter.search( + ",".join( + [ + user_ou, + root_dn, + ] + ), + full_role_filter, + attributes=["cn"], + dereference_aliases=DEREF_NEVER, + ) + except Exception as e: + log.exception("Failed to search for roles") + raise e + + roles_found = sorted( + set({entry.entry_dn.split(",")[1].split("=")[1] for entry in ldap_connection_role_filter.entries}) + ) + log.debug("users found from roles filter: ") + log.debug(roles_found) + + ldap_connection_role_filter.unbind() + + # generate a list of matches in roles and users + matched_users = set(users_found) & set(roles_found) + log.debug("matched users: ") + log.debug(matched_users) + + # cartesian_product = [(user, role) for user in matched_users for role in roles] + + cartesian_product = list( + product( + matched_users, + roles, + ) + ) + log.debug("cartesian product: ") + log.debug(cartesian_product) + + try: + ldap_connection_action = ldap_connect( + env.vars.get("LDAP_HOST"), + env.vars.get("LDAP_USER"), + env.secrets.get("LDAP_BIND_PASSWORD"), + ) + except Exception as e: + log.exception("Failed to connect to LDAP") + raise e + + for item in cartesian_product: + if add: + try: + ldap_connection_action.add( + f"cn={item[1]},cn={item[0]},{user_ou},{root_dn}", + attributes={ + "cn": item[1], + "aliasedObjectName": f"cn={item[1]},cn=ndRoleCatalogue,{user_ou},{root_dn}", + "objectClass": [ + "NDRoleAssociation", + "alias", + "top", + ], + }, + ) + except Exception as e: + log.exception(f"Failed to add role '{item[1]}' to user '{item[0]}'") + raise e + if ldap_connection_action.result["result"] == 0: + log.info(f"Successfully added role '{item[1]}' to user '{item[0]}'") + elif ldap_connection_action.result["result"] == 68: + log.info(f"Role '{item[1]}' already present for user '{item[0]}'") + else: + log.e(f"Failed to add role '{item[1]}' to user '{item[0]}'") + log.debug(ldap_connection_action.result) + elif remove: + ldap_connection_action.delete(f"cn={item[1]},cn={item[0]},{user_ou},{root_dn}") + if ldap_connection_action.result["result"] == 0: + log.info(f"Successfully removed role '{item[1]}' from user '{item[0]}'") + elif ldap_connection_action.result["result"] == 32: + log.info(f"Role '{item[1]}' already absent for user '{item[0]}'") + else: + log.error(f"Failed to remove role '{item[1]}' from user '{item[0]}'") + log.debug(ldap_connection_action.result) + else: + log.error("No action specified") + + if update_notes: + connection = cli.database.connection() + log.debug("Created database cursor successfully") + for user in matched_users: + try: + update_sql = """ + UPDATE USER_ SET LAST_UPDATED_DATETIME=CURRENT_DATE, + LAST_UPDATED_USER_ID=4 WHERE UPPER(DISTINGUISHED_NAME)=UPPER(:user_dn) + """ + update_cursor = connection.cursor() + update_cursor.execute( + update_sql, + [user], + ) + update_cursor.close() + + insert_sql = """ + INSERT INTO USER_NOTE ( + USER_NOTE_ID, + USER_ID, + LAST_UPDATED_USER_ID, + LAST_UPDATED_DATETIME, + NOTES + ) + SELECT + user_note_id_seq.nextval, + USER_ID, + 4, + sysdate, + :user_note + FROM + USER_ + WHERE + UPPER(DISTINGUISHED_NAME) = UPPER(:user_dn) + """ + insert_cursor = connection.cursor() + insert_cursor.setinputsizes(user_note=oracledb.CLOB) + insert_cursor.execute( + insert_sql, + user_note=user_note, + user_dn=user, + ) + insert_cursor.close() + + log.info(f"Updated notes for user {user}") + connection.commit() + log.info("Committed changes to database successfully") + except: + log.exception(f"Failed to update notes for user {user}") + connection.close() + + +######################################### +# Deactivate CRC User Accounts +######################################### + + +def deactivate_crc_users( + user_ou, + root_dn, +): + log.info("Deactivating CRC users") + ldap_connection = ldap_connect( + env.vars.get("LDAP_HOST"), + env.vars.get("LDAP_USER"), + env.secrets.get("LDAP_BIND_PASSWORD"), + ) + + user_filter = "(userSector=private)(!(userSector=public))(!(endDate=*))(objectclass=NDUser)" + + home_areas = [ + [ + "C01", + "C02", + "C03", + "C04", + "C05", + "C06", + "C07", + "C08", + "C09", + "C10", + "C11", + "C12", + "C13", + "C14", + "C15", + "C16", + "C17", + "C18", + "C19", + "C20", + "C21", + ] + ] + + found_users = [] + for home_area in home_areas: + ldap_connection.search( + ",".join( + [ + user_ou, + root_dn, + ] + ), + f"(&(userHomeArea={home_area})(!(cn={home_area})){user_filter})", + attributes=["dn"], + ) + + found_users.append(entry.entry_dn for entry in ldap_connection.entries) + + ldap_connection.search( + ",".join( + [ + user_ou, + root_dn, + ] + ), + f"(&(!(userHomeArea=*)){user_filter})", + attributes=["dn"], + ) + found_users_no_home_area = [entry.entry_dn for entry in ldap_connection.entries] + + all_users = found_users + found_users_no_home_area + + date_str = f"{datetime.now().strftime('%Y%m%d')}000000Z" + + for user in all_users: + ldap_connection.modify( + user, + { + "endDate": [ + ( + MODIFY_REPLACE, + [date_str], + ) + ] + }, + ) + + connection = cli.database.connection() + for user_dn in all_users: + try: + update_sql = ( + f"UPDATE USER_ SET END_DATE=TRUNC(CURRENT_DATE) WHERE UPPER(DISTINGUISHED_NAME)=UPPER(:user_dn)" + ) + update_cursor = connection.cursor() + update_cursor.execute( + update_sql, + [user_dn], + ) + update_cursor.close() + log.info(f"Updated END_DATE for user {user_dn}") + connection.commit() + log.info("Committed changes to database successfully") + except: + log.exception(f"Failed to update END_DATE for user {user_dn}") + connection.close() diff --git a/cli/logger.py b/cli/logger.py new file mode 100644 index 0000000..a76bb6a --- /dev/null +++ b/cli/logger.py @@ -0,0 +1,83 @@ +import logging +import cli.env + + +def configure_logging(): + class SensitiveFormatter(logging.Formatter): + """Formatter that removes secrets from log messages.""" + + def __init__( + self, + format_str=None, + datefmt_str=None, + ): + super().__init__( + fmt=format_str, + datefmt=datefmt_str, + ) + self._secrets_set = set( + cli.env.secrets.values() + ) # Retrieve secrets set here + self.default_msec_format = "%s.%03d" + + def _filter( + self, + s, + ): + redacted = " ".join( + [ + "*" * len(string) if string in self._secrets_set else string + for string in s.split(" ") + ] + ) + + return redacted + + def format( + self, + record, + ): + original = super().format(record) + return self._filter(original) + + print("configure_logging") + """Configure logging based on environment variables.""" + format = ( + cli.env.vars.get("LOG_FORMAT") + or "%(asctime)s.%(msecs)03d - %(levelname)s: %(message)s" + ) + datefmt = cli.env.vars.get("LOG_DATE_FORMAT") or "%Y-%m-%d %H:%M:%S" + + log = logging.getLogger(__name__) + + if logging.root.hasHandlers(): + logging.root.handlers = [] + + handler = logging.StreamHandler() + handler.setFormatter( + SensitiveFormatter( + format_str=format, + datefmt_str=datefmt, + ) + ) + logging.root.addHandler(handler) + if cli.env.vars.get("LOG_LEVEL") == "DEBUG": + print("DEBUG") + log.setLevel(logging.DEBUG) + elif cli.env.vars.get("LOG_LEVEL") == "WARNING": + print("WARNING") + log.setLevel(logging.WARNING) + elif cli.env.vars.get("LOG_LEVEL") == "ERROR": + print("ERROR") + log.setLevel(logging.ERROR) + elif cli.env.vars.get("LOG_LEVEL") == "INFO": + print("INFO") + log.setLevel(logging.INFO) + else: + print("No LOG_LEVEL set, defaulting to INFO") + print("INFO") + log.setLevel(logging.INFO) + return True + + +log = logging.getLogger(__name__) diff --git a/cli/template/__init__.py b/cli/template/__init__.py new file mode 100644 index 0000000..0a02f78 --- /dev/null +++ b/cli/template/__init__.py @@ -0,0 +1,55 @@ +import os.path + +import jinja2 +from pathlib import ( + Path, +) + + +def render(template_path, **kwargs): + parent_path = Path(template_path).parent + env = jinja2.Environment( + loader=jinja2.FileSystemLoader(searchpath=parent_path), + autoescape=True, + ) + template = env.get_template(Path(template_path).name) + return template.render(**kwargs) + + +def save( + rendered_text, + template_path, + rendered_dir="./rendered/", +): + # create rendered_dir if it doesn't exist + if not Path.exists(Path(rendered_dir)): + Path.mkdir(Path(rendered_dir)) + # create the directory structure for the template file if it doesn't exist + if not Path.exists( + Path( + os.path.join( + rendered_dir, + template_path, + ) + ).parent + ): + Path.mkdir( + Path( + os.path.join( + rendered_dir, + template_path, + ) + ).parent + ) + file = Path( + os.path.join( + rendered_dir, + template_path.replace( + ".j2", + "", + ), + ) + ) + file.touch(exist_ok=True) + file.write_text(rendered_text) + return file diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..67caa7b --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,16 @@ +[tool.black] +# PEP-8 line length is not very practical +line-length = 88 +include = '\.pyi?$' +exclude = ''' +/( + \.git + | \.py_cache + | \venv +)/ +''' + +[tool.codespell] +skip = './venv,./rbac,./rendered' +count = '' +quiet-level = 3 \ No newline at end of file diff --git a/readme.md b/readme.md index eaa445a..3475378 100644 --- a/readme.md +++ b/readme.md @@ -1,9 +1,9 @@ # Quick start guide - ## Environment variables -Variables are picked up from the environment, or can be specified in a `.env` file in the current directory (at the same level as the file `setup.py`) +Variables are picked up from the environment, or can be specified in a `.env` file in the current directory (at the same +level as the file `setup.py`) See `cli/config.py` for a list of variables. ## Installation for development purposes @@ -20,10 +20,20 @@ See `cli/config.py` for a list of variables. `pip install git+https://github.com/ministryofjustice/hmpps-ldap-automation-cli.git` -Optionally append `@`, `@` or -`@` to the end of the url to install a specific +Optionally append `@`, `@` or +`@` to the end of the url to install a specific commit ## Usage `ldap-automation --help` + +# Dev + +## pre-commit + +This project uses pre-commit to run a number of checks on the code before it is committed. +To install the pre-commit hooks run: + +`pre-commit install` + diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..77bfeb0 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,14 @@ +click==8.1.6 +ldap3 +oracledb==1.4 +ansible-runner +PyGithub +GitPython +pyjwt +python-dotenv +python-ldap +Jinja2 +pre-commit +black +flake8 +codespell \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 95ddc82..d54edec 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,12 @@ click==8.1.6 -ldap3 -oracledb==1.2.2 +ldap3~=2.9.1 +oracledb==1.4 ansible-runner PyGithub -GitPython -pyjwt -python-dotenv \ No newline at end of file +GitPython==3.1.37 +pyjwt~=2.8.0 +python-dotenv~=1.0.0 +Jinja2~=3.1.2 +python-ldap +requests~=2.31.0 +setuptools~=68.2.2 diff --git a/setup.py b/setup.py index 4821b95..b28c4dc 100644 --- a/setup.py +++ b/setup.py @@ -1,11 +1,24 @@ -from setuptools import setup, find_packages +from setuptools import ( + setup, + find_packages, +) + +# Read requirements from requirements.txt +with open("requirements.txt") as f: + requirements = f.read().splitlines() + +standard_pkgs = [r for r in requirements if not r.startswith("git+")] +git_pkgs = [r for r in requirements if r.startswith("git+")] +formatted_git_pkgs = [ + f"{git_pkg.split('/')[-1].split('.git@')[0]} @ {git_pkg}" for git_pkg in git_pkgs +] +all_reqs = standard_pkgs + formatted_git_pkgs setup( name="ldap-automation", version="0.1", packages=find_packages(), - # install_requires=["Click", "ldap3", "oracledb"], - install_requires=["Click", "ldap3"], + install_requires=all_reqs, entry_points=""" [console_scripts] ldap-automation=cli:main_group