diff --git a/.gitignore b/.gitignore index 886a8c8..6887167 100644 --- a/.gitignore +++ b/.gitignore @@ -159,11 +159,4 @@ cython_debug/ # option (not recommended) you can uncomment the following to ignore the entire idea folder. .idea/ # VSCode Config -.vscode - -.secrets -.vars -/rbac - -*.ldif -*.ldif.j2 \ No newline at end of file +.vscode \ No newline at end of file diff --git a/cli/__init__.py b/cli/__init__.py index 9e60271..fca6d0f 100644 --- a/cli/__init__.py +++ b/cli/__init__.py @@ -1,33 +1,28 @@ import click -import cli.ldap.add_roles_to_username, cli.ldap.rbac +from cli import ldap from cli import git -import cli.env - @click.group() def main_group(): pass - @click.command() @click.option("--user-ou", help="OU to add users to, defaults to ou=Users", default="ou=Users") @click.option("--root-dn", help="Root DN to add users to", default="dc=moj,dc=com") @click.argument("user-role-list", required=True) def add_roles_to_users(user_ou, root_dn, user_role_list): - cli.ldap.add_roles_to_username.process_user_roles_list(user_role_list, user_ou, root_dn) + ldap.process_user_roles_list(user_role_list, user_ou, root_dn) @click.command() -@click.option("--rbac-repo-tag", help="RBAC repo tag to use", default="master") -def rbac_uplift(rbac_repo_tag): - cli.ldap.rbac.main(rbac_repo_tag) - +def git_test(): + git.dl_test() # from cli.ldap import test main_group.add_command(add_roles_to_users) -main_group.add_command(rbac_uplift) +main_group.add_command(git_test) if __name__ == "__main__": main_group() diff --git a/cli/ansible/__init__.py b/cli/ansible/__init__.py new file mode 100644 index 0000000..98c7787 --- /dev/null +++ b/cli/ansible/__init__.py @@ -0,0 +1,2 @@ +import ansible_runner + diff --git a/cli/config.py b/cli/config.py new file mode 100644 index 0000000..5a7343a --- /dev/null +++ b/cli/config.py @@ -0,0 +1,17 @@ +import os + +from dotenv import load_dotenv + +load_dotenv() + +ldap_host = os.getenv("LDAP_HOST") +ldap_user = os.getenv("LDAP_USER") +ldap_password = os.getenv("LDAP_PASSWORD") +db_user = os.getenv("DB_USER") +db_password = os.getenv("DB_PASSWORD") +db_host = os.getenv("DB_HOST") +db_port = os.getenv("DB_PORT") +db_service_name = os.getenv("DB_SERVICE_NAME") +gh_app_id = os.getenv("GH_APP_ID") +gh_private_key = os.getenv("GH_PRIVATE_KEY") +gh_installation_id = os.getenv("GH_INSTALLATION_ID") \ No newline at end of file diff --git a/cli/env.py b/cli/env.py deleted file mode 100644 index 2a96160..0000000 --- a/cli/env.py +++ /dev/null @@ -1,48 +0,0 @@ -import os - -from dotenv import dotenv_values - -import ast - -# ldap_host = os.getenv("LDAP_HOST") -# ldap_user = os.getenv("LDAP_USER") -# ldap_password = os.getenv("LDAP_PASSWORD") -# db_user = os.getenv("DB_USER") -# db_password = os.getenv("DB_PASSWORD") -# db_host = os.getenv("DB_HOST") -# db_port = os.getenv("DB_PORT") -# db_service_name = os.getenv("DB_SERVICE_NAME") -# gh_app_id = os.getenv("GH_APP_ID") -# gh_private_key = os.getenv("GH_PRIVATE_KEY") -# gh_installation_id = os.getenv("GH_INSTALLATION_ID") - - -vars = { - **{ - key.replace("VAR_", "").replace("_DICT", ""): ast.literal_eval(val) if "DICT" in key else val - for key, val in dotenv_values(".vars").items() - if val is not None - }, # load development variables - **{ - key.replace("VAR_", "").replace("_DICT", ""): ast.literal_eval(val) if "DICT" in key else val - for key, val in os.environ.items() - if key.startswith("VAR_") and val is not None - }, -} -# loads all environment variables starting with SECRET_ into a dictionary -secrets = { - **{ - key.replace("SECRET_", "").replace("_DICT", "").replace("SSM_", ""): ast.literal_eval(val) - if "_DICT" in key - else val - for key, val in dotenv_values(".secrets").items() - if val is not None - }, - **{ - key.replace("SECRET_", "").replace("_DICT", "").replace("SSM_", ""): ast.literal_eval(val) - if "DICT" in key - else val - for key, val in os.environ.items() - if key.startswith("SECRET_") or key.startswith("SSM_") and val is not None - }, -} diff --git a/cli/git/__init__.py b/cli/git/__init__.py index 5378e65..ad06a28 100644 --- a/cli/git/__init__.py +++ b/cli/git/__init__.py @@ -4,37 +4,47 @@ import time import requests import logging -from cli import env - - +from cli import config def get_access_token(app_id, private_key, installation_id): # Create a JSON Web Token (JWT) using the app's private key now = int(time.time()) - payload = {"iat": now, "exp": now + 600, "iss": app_id} + payload = { + "iat": now, + "exp": now + 600, + "iss": app_id + } jwt_token = jwt.encode(payload, private_key, algorithm="RS256") # Exchange the JWT for an installation access token - headers = {"Authorization": f"Bearer {jwt_token}", "Accept": "application/vnd.github.v3+json"} - response = requests.post( - f"https://api.github.com/app/installations/{installation_id}/access_tokens", headers=headers - ) + headers = { + "Authorization": f"Bearer {jwt_token}", + "Accept": "application/vnd.github.v3+json" + } + response = requests.post(f"https://api.github.com/app/installations/{installation_id}/access_tokens", + headers=headers) # extract the token from the response access_token = response.json().get("token") return access_token - -def get_repo(url, depth="1", branch_or_tag="master", token=None, auth_type="x-access-token", dest_name="repo"): +def get_repo(url, token=None, auth_type="x-access-token", dest_name="repo"): # if there is an @ in the url, assume auth is already specified - multi_options = ["--depth " + depth, "--branch " + branch_or_tag] - if "@" in url: - logging.info("auth already specified in url") - return Repo.clone_from(url, dest_name, multi_options=multi_options) + if '@' in url: + logging.info('auth already specified in url') + return Repo.clone_from(url, dest_name) # if there is a token, assume auth is required and use the token and auth_type elif token: templated_url = f'https://{auth_type}:{token}@{url.split("//")[1]}' - logging.info(f"cloning with token: {templated_url}") - return Repo.clone_from(templated_url, dest_name, multi_options=multi_options) + logging.info(f'cloning with token: {templated_url}') + return Repo.clone_from(templated_url, dest_name) # if there is no token, assume auth is not required and clone without else: - logging.info("cloning without auth") - return Repo.clone_from(url, dest_name, multi_options=multi_options) + logging.info('cloning without auth') + return Repo.clone_from(url, dest_name) +def dl_test(): + app_id = config.gh_app_id + private_key = config.gh_private_key + installation_id = config.gh_installation_id + url = 'https://github.com/ministryofjustice/hmpps-delius-pipelines.git' + token = get_access_token(app_id, private_key, installation_id) + repo = get_repo(url, token=token, dest_name='delius-pipelines') + print(repo) \ No newline at end of file diff --git a/cli/ldap/__init__.py b/cli/ldap/__init__.py index a969c09..126deb7 100644 --- a/cli/ldap/__init__.py +++ b/cli/ldap/__init__.py @@ -1,12 +1,14 @@ from ldap3 import Server, Connection, ALL -from logging import log - # import oracledb +import logging + +logging.basicConfig(level=logging.DEBUG) + + def ldap_connect(ldap_host, ldap_user, ldap_password): - server = Server(ldap_host, get_info=ALL) return Connection( - server=server, user=ldap_user, password=ldap_password, auto_bind="NO_TLS", authentication="SIMPLE" + server=ldap_host, user=ldap_user, password=ldap_password, auto_bind="NO_TLS", authentication="SIMPLE" ) diff --git a/cli/ldap/add_roles_to_username.py b/cli/ldap/add_roles_to_username.py index c1736a9..0ae4396 100644 --- a/cli/ldap/add_roles_to_username.py +++ b/cli/ldap/add_roles_to_username.py @@ -1,22 +1,16 @@ -from cli.logging import log -from cli import env +import logging + +from cli import config from cli.ldap import ldap_connect def parse_user_role_list(user_role_list): - # The format of the list should be a pipe separated list of username and role lists, - # where the username and role list is separated by a comma character, - # and the roles are separated by a semi-colon: - # username1,role1;role2;role3|username2,role1;role2 - return {user.split(",")[0]: user.split(",")[1].split(";") for user in user_role_list.split("|")} def add_roles_to_user(username, roles, user_ou="ou=Users", root_dn="dc=moj,dc=com"): - log.info(f"Adding roles {roles} to user {username}") - ldap_connection = ldap_connect( - env.vars.get("LDAP_HOST"), env.vars.get("LDAP_USER"), env.secrets.get("LDAP_BIND_PASSWORD") - ) + logging.info(f"Adding roles {roles} to user {username}") + ldap_connection = ldap_connect(config.ldap_host, config.ldap_user, config.ldap_password) for role in roles: ldap_connection.add( f"cn={role},cn={username},{user_ou},{root_dn}", @@ -36,7 +30,6 @@ def add_roles_to_user(username, roles, user_ou="ou=Users", root_dn="dc=moj,dc=co def process_user_roles_list(user_role_list, user_ou="ou=Users", root_dn="dc=moj,dc=com"): - log.info(f"secrets: {env.secrets}") user_roles = parse_user_role_list(user_role_list) for user, roles in user_roles.items(): add_roles_to_user(user, roles, user_ou, root_dn) diff --git a/cli/ldap/rbac.py b/cli/ldap/rbac.py deleted file mode 100644 index 9aecaa3..0000000 --- a/cli/ldap/rbac.py +++ /dev/null @@ -1,256 +0,0 @@ -import re - -import ldap3.utils.hashed -from cli.ldap import ldap_connect -from cli import env -import cli.git as git -import glob -from cli.logging import log -from pathlib import Path -import cli.template -from ldif import LDIFParser - - -def get_repo(repo_tag="master"): - app_id = env.vars.get("GH_APP_ID") - private_key = env.vars.get("GH_PRIVATE_KEY") - installation_id = env.vars.get("GH_INSTALLATION_ID") - # url = 'https://github.com/ministryofjustice/hmpps-delius-pipelines.git' - url = "https://github.com/ministryofjustice/hmpps-ndelius-rbac.git" - token = git.get_access_token(app_id, private_key, installation_id) - try: - repo = git.get_repo(url, token=token, dest_name="rbac", branch_or_tag=repo_tag) - return repo - except Exception as e: - log.exception(e) - return None - - -def prep_for_templating(files, strings=None): - if strings is None: - strings = env.vars.get("RBAC_SUBSTITUTIONS") - # get a list of files - # print(strings) - # print(type(strings)) - for file_path in files: - file = Path(file_path) - for k, v in strings.items(): - print("replacing", k, "with", v, "in", file_path) - # print( - # file.read_text().replace(k, v), - # ) - file.write_text( - file.read_text().replace(k, v), - ) - - -def template_rbac(files): - hashed_pwd_admin_user = ldap3.utils.hashed.hashed(ldap3.HASHED_SALTED_SHA, env.secrets.get("LDAP_ADMIN_PASSWORD")) - rendered_files = [] - for file in files: - rendered_text = cli.template.render( - file, - ldap_config=env.vars.get("LDAP_CONFIG"), - bind_password_hash=hashed_pwd_admin_user, - secrets=env.secrets, - oasys_password=env.secrets.get("OASYS_PASSWORD"), - environment_name=env.vars.get("ENVIRONMENT_NAME"), - project_name=env.vars.get("PROJECT_NAME"), - ) - rendered_file = cli.template.save(rendered_text, file) - rendered_files.append(rendered_file) - return rendered_files - - -def context_ldif(rendered_files): - context_file = [file for file in rendered_files if "context" in Path(file).name] - for file in context_file: - parser = LDIFParser(open(file, "rb"), strict=False) - for dn, record in parser.parse(): - print("got entry record: %s" % dn) - print(record) - ldap_connection = ldap_connect( - env.vars.get("LDAP_HOST"), env.vars.get("LDAP_USER"), env.secrets.get("LDAP_BIND_PASSWORD") - ) - ldap_connection.add(dn, attributes=record) - if any(result not in [0, 68] for result in ldap_connection.result["result"]): - log.debug(ldap_connection.result) - log.debug(ldap_connection.response) - raise Exception(f"Failed to add context {dn}, status: {ldap_connection.result['result']}") - - -def group_ldifs(rendered_files): - # connect to ldap - ldap_connection = ldap_connect( - env.vars.get("LDAP_HOST"), env.vars.get("LDAP_USER"), env.secrets.get("LDAP_BIND_PASSWORD") - ) - group_files = [file for file in rendered_files if "groups" in Path(file).name] - # loop through the group files - for file in group_files: - # parse the ldif into dn and record - parser = LDIFParser(open(file, "rb"), strict=False) - # loop through the records - for dn, record in parser.parse(): - print("got entry record: %s" % dn) - print(record) - # add the record to ldap - ldap_connection.add(dn, attributes=record) - if record.get("description"): - print("updating description") - ldap_connection.modify(dn, {"description": [(ldap3.MODIFY_REPLACE, record["description"])]}) - if any(result not in [0, 68] for result in ldap_connection.result["result"]): - log.debug(ldap_connection.result) - log.debug(ldap_connection.response) - raise Exception( - f"Failed to update description for group {dn}, status: {ldap_connection.result['result']}" - ) - - -def policy_ldifs(rendered_files): - # connect to ldap - ldap_connection = ldap_connect( - env.vars.get("LDAP_HOST"), env.vars.get("LDAP_USER"), env.secrets.get("LDAP_BIND_PASSWORD") - ) - policy_files = [file for file in rendered_files if "policy" in Path(file).name] - - # first, delete the policies - ldap_connection.delete("ou=Policies," + env.vars.get("LDAP_CONFIG").get("base_root")) - - # loop through the policy files - for file in policy_files: - # parse the ldif into dn and record - parser = LDIFParser(open(file, "rb"), strict=False) - # loop through the records - for dn, record in parser.parse(): - print("got entry record: %s" % dn) - # print(record) - # add the record to ldap - ldap_connection.add(dn, attributes=record) - if any(result not in [0, 68] for result in ldap_connection.result["result"]): - log.debug(ldap_connection.result) - log.debug(ldap_connection.response) - raise Exception(f"Failed to add policy {dn}, status: {ldap_connection.result['result']}") - - -def role_ldifs(rendered_files): - # connect to ldap - ldap_connection = ldap_connect( - env.vars.get("LDAP_HOST"), env.vars.get("LDAP_USER"), env.secrets.get("LDAP_BIND_PASSWORD") - ) - role_files = [file for file in rendered_files if "nd_role" in Path(file).name] - - # first, delete the roles - ldap_connection.delete("cn=ndRoleCatalogue," + env.vars.get("LDAP_CONFIG").get("base_users")) - ldap_connection.delete("cn=ndRoleGroups," + env.vars.get("LDAP_CONFIG").get("base_users")) - - # ensure boolean values are Uppercase.. - # (not yet implemented, probably not needed) - - # loop through the role files - for file in role_files: - # parse the ldif into dn and record - parser = LDIFParser(open(file, "rb"), strict=False) - # loop through the records - for dn, record in parser.parse(): - print("got entry record: %s" % dn) - # print(record) - # add the record to ldap - ldap_connection.add(dn, attributes=record) - if any(result not in [0, 68] for result in ldap_connection.result["result"]): - log.debug(ldap_connection.result) - log.debug(ldap_connection.response) - raise Exception(f"Failed to add role {dn}, status: {ldap_connection.result['result']}") - - -# not complete!! -# see https://github.com/ministryofjustice/hmpps-delius-pipelines/blob/master/components/delius-core/playbooks/rbac/import_schemas.yml -def schema_ldifs(rendered_files): - # connect to ldap - ldap_connection = ldap_connect( - env.vars.get("LDAP_HOST"), env.vars.get("LDAP_USER"), env.secrets.get("LDAP_BIND_PASSWORD") - ) - - schema_files = [file for file in rendered_files if "delius.ldif" or "pwm.ldif" in Path(file).name] - - # loop through the schema files - for file in schema_files: - # parse the ldif into dn and record - parser = LDIFParser(open(file, "rb"), strict=False) - # loop through the records - for dn, record in parser.parse(): - print("got entry record: %s" % dn) - # print(record) - # add the record to ldap - ldap_connection.add(dn, attributes=record) - if any(result not in [0, 68] for result in ldap_connection.result["result"]): - log.debug(ldap_connection.result) - log.debug(ldap_connection.response) - raise Exception(f"Failed to add schema {dn}, status: {ldap_connection.result['result']}") - - -def user_ldifs(rendered_files): - # connect to ldap - ldap_connection = ldap_connect( - env.vars.get("LDAP_HOST"), env.vars.get("LDAP_USER"), env.secrets.get("LDAP_BIND_PASSWORD") - ) - user_files = [file for file in rendered_files if "-users" in Path(file).name] - - # first, delete the users - for file in user_files: - # parse the ldif into dn and record - parser = LDIFParser(open(file, "rb"), strict=False) - # loop through the records - for dn, record in parser.parse(): - print("got entry record: %s" % dn) - # print(record) - # add the record to ldap - ldap_connection.delete(dn) - if any(result not in [0, 68] for result in ldap_connection.result["result"]): - log.debug(ldap_connection.result) - log.debug(ldap_connection.response) - raise Exception(f"Failed to delete user {dn}, status: {ldap_connection.result['result']}") - - # loop through the user files - for file in user_files: - # parse the ldif into dn and record - parser = LDIFParser(open(file, "rb"), strict=False) - # loop through the records - for dn, record in parser.parse(): - print("got entry record: %s" % dn) - # print(record) - # add the record to ldap - ldap_connection.add(dn, attributes=record) - if any(result not in [0, 68] for result in ldap_connection.result["result"]): - log.debug(ldap_connection.result) - log.debug(ldap_connection.response) - raise Exception(f"Failed to add user {dn}, status: {ldap_connection.result['result']}") - - -def main(rbac_repo_tag): - repo = get_repo(rbac_repo_tag) - print(env.vars.get("RBAC_SUBSTITUTIONS")) - dir = "./rbac" - - files = [ - file - for file in glob.glob(f"{dir}/**/*", recursive=True) - if Path(file).is_file() and Path(file).name.endswith(".ldif") or Path(file).name.endswith(".j2") - ] - - prep_for_templating(files) - rendered_files = template_rbac(files) - context_ldif(rendered_files) - policy_ldifs(rendered_files) - # schema_ldifs(files) probably not needed, but check! - role_ldifs(rendered_files) - group_ldifs(rendered_files) - user_ldifs(rendered_files) - - parser = LDIFParser(open("./rendered/rbac/context.ldif", "rb"), strict=False) - for dn, record in parser.parse(): - print("got entry record: %s" % dn) - print(record) - ldap_connection = ldap_connect( - env.vars.get("LDAP_HOST"), env.vars.get("LDAP_USER"), env.secrets.get("LDAP_BIND_PASSWORD") - ) - ldap_connection.add(dn, attributes=record) diff --git a/cli/ldap/test.py b/cli/ldap/test.py new file mode 100644 index 0000000..1070de6 --- /dev/null +++ b/cli/ldap/test.py @@ -0,0 +1,15 @@ +from cli import config +from cli.ldap import ldap_connect +from ldap3 import LEVEL +import click + + +def test_search(): + ldap_connection = ldap_connect(config.ldap_host, config.ldap_user, config.ldap_password) + ldap_connection.search("dc=moj,dc=com", "(objectClass=*)", search_scope=LEVEL, attributes=["*"], time_limit=120) + print(ldap_connection.entries) + + +@click.command() +def test(): + test_search() diff --git a/cli/logging.py b/cli/logging.py deleted file mode 100644 index 0b78a11..0000000 --- a/cli/logging.py +++ /dev/null @@ -1,22 +0,0 @@ -import logging -from cli.env import secrets - - -class SensitiveFormatter(logging.Formatter): - """Formatter that removes secrets from log messages.""" - - @staticmethod - def _filter(s): - secrets_set = set(secrets.values()) - redacted = " ".join([s.replace(secret, "*" * len(secret)) for secret in secrets_set if secret is not None]) - return redacted - - def format(self, record): - original = logging.Formatter.format(self, record) - return self._filter(original) - - -logging.basicConfig(level=logging.DEBUG) -log = logging.getLogger(__name__) -for handler in log.root.handlers: - handler.setFormatter(SensitiveFormatter()) diff --git a/cli/template/__init__.py b/cli/template/__init__.py deleted file mode 100644 index 55c42c1..0000000 --- a/cli/template/__init__.py +++ /dev/null @@ -1,24 +0,0 @@ -import os.path - -import jinja2 -from pathlib import Path - - -def render(template_path, **kwargs): - parent_path = Path(template_path).parent - env = jinja2.Environment(loader=jinja2.FileSystemLoader(searchpath=parent_path), autoescape=True) - template = env.get_template(Path(template_path).name) - return template.render(**kwargs) - - -def save(rendered_text, template_path, rendered_dir="./rendered/"): - # create rendered_dir if it doesn't exist - if not Path.exists(Path(rendered_dir)): - Path.mkdir(Path(rendered_dir)) - # create the directory structure for the template file if it doesn't exist - if not Path.exists(Path(os.path.join(rendered_dir, template_path)).parent): - Path.mkdir(Path(os.path.join(rendered_dir, template_path)).parent) - file = Path(os.path.join(rendered_dir, template_path.replace(".j2", ""))) - file.touch(exist_ok=True) - file.write_text(rendered_text) - return file diff --git a/requirements.txt b/requirements.txt index 4149531..95ddc82 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,6 +5,4 @@ ansible-runner PyGithub GitPython pyjwt -python-dotenv -Jinja2 -git+https://github.com/abilian/ldif.git@4.2.0 \ No newline at end of file +python-dotenv \ No newline at end of file