Merge pull request #93 from Karandash8/92-refactor-make-argocd-fly-ap…

…plications

Refactor applications logic

README.md

@@ -95,6 +95,7 @@ envs:
       <application_name>:
         app_deployer: <bootstrap_application>  ## application that will deploy this application
         app_deployer_env: <environment_name>  ## (OPTIONAL) for multi-environments with single ArgoCD deployment
+        non_k8s_files_to_render: [<filename>]  ## (OPTIONAL) list of files to render that are not Kubernetes resources (e.g., values.yml)
         project: <project_name>  ## ArgoCD project name
         destination_namespace: <namespace>  ## default namespace where the application resources will be deployed
 vars:
@@ -136,7 +137,7 @@ vars:
 ```
 
 ### Variables in `config.yml`
-Variables can be referenced in the configuration file (including in the application parameters section) using the following syntax:
+Variables can be referenced in the configuration file using the following syntax:
 ```${var_name}``` and  ```${var_name[dict_key][...]}```.
 
 Variables can also be used as substring values:
@@ -147,7 +148,7 @@ To include file content in the current Jinja2 template, use the following block:
 
 ```
 {%- filter indent(width=4) %}
-{% include_raw 'app_4/files/file.json' %}
+{% include_raw 'files/file.json' %}
 {% endfilter %}
 ```
 
@@ -158,7 +159,7 @@ To render a template in the current jinja2 template, use the following block:
 
 ```
 {%- filter indent(width=4) %}
-{% include 'app_5/files/file.json.j2' %}
+{% include 'files/file.json.j2' %}
 {% endfilter %}
 ```
 

dev_requirements.txt

@@ -2,5 +2,6 @@ setuptools
 build==0.10.0
 twine==4.0.2
 pytest==7.4.0
+pytest-asyncio==0.23.8
 yelp-gprof2dot==1.2.0
 flake8==7.0.0

make_argocd_fly/config.py

@@ -41,13 +41,13 @@ def get_tmp_dir(self) -> str:
     return get_abs_path(self.root_dir, self.tmp_dir, allow_missing=True)
 
   def get_envs(self) -> dict:
-    if not self.envs:
+    if not isinstance(self.envs, dict):
       log.error('Config was not initialized.')
       raise Exception
     return self.envs
 
   def get_vars(self) -> dict:
-    if not self.envs:
+    if not isinstance(self.vars, dict):
       log.error('Config was not initialized.')
       raise Exception
     return self.vars
@@ -71,6 +71,23 @@ def get_app_vars(self, env_name: str, app_name: str) -> dict:
 
     return envs[env_name]['apps'][app_name]['vars'] if 'vars' in envs[env_name]['apps'][app_name] else {}
 
+  def get_app_params(self, env_name: str, app_name: str) -> dict:
+    envs = self.get_envs()
+    if env_name not in envs:
+      log.error('Environment {} is not defined'.format(env_name))
+      raise Exception
+
+    if app_name not in envs[env_name]['apps']:
+      log.error('Application {} is not defined in environment {}'.format(app_name, env_name))
+      raise Exception
+
+    params = {}
+    for key, value in envs[env_name]['apps'][app_name].items():
+      if key != 'vars':
+        params[key] = value
+
+    return params
+
 
 config = Config()
 

make_argocd_fly/exceptions.py

@@ -0,0 +1,2 @@
+class MissingSourceResourcesError(Exception):
+  pass

make_argocd_fly/main.py

@@ -11,9 +11,8 @@
 from make_argocd_fly.cli_args import populate_cli_args, get_cli_args
 from make_argocd_fly.config import read_config, get_config, LOG_CONFIG_FILE, CONFIG_FILE, \
   SOURCE_DIR, OUTPUT_DIR, TMP_DIR
-from make_argocd_fly.utils import multi_resource_parser, generate_filename, latest_version_check
-from make_argocd_fly.resource import ResourceViewer, ResourceWriter
-from make_argocd_fly.application import application_factory
+from make_argocd_fly.utils import latest_version_check
+from make_argocd_fly.application import workflow_factory, Application
 
 
 logging.basicConfig(level='INFO')
@@ -32,12 +31,11 @@ def init_logging(loglevel: str) -> None:
     pass
 
 
-def create_applications(render_apps, render_envs):
+async def generate() -> None:
   config = get_config()
-
-  log.info('Reading source directory')
-  source_viewer = ResourceViewer(config.get_source_dir())
-  source_viewer.build()
+  cli_args = get_cli_args()
+  render_apps = cli_args.get_render_apps()
+  render_envs = cli_args.get_render_envs()
 
   apps_to_render = render_apps.split(',') if render_apps is not None else []
   envs_to_render = render_envs.split(',') if render_envs is not None else []
@@ -52,47 +50,17 @@ def create_applications(render_apps, render_envs):
       if apps_to_render and app_name not in apps_to_render:
         continue
 
-      app_viewer = source_viewer.get_element(app_name)
-      apps.append(application_factory(app_viewer, app_name, env_name))
-
-  return apps
-
-
-async def generate() -> None:
-  cli_args = get_cli_args()
-  config = get_config()
-
-  apps = create_applications(cli_args.get_render_apps(), cli_args.get_render_envs())
+      workflow = await workflow_factory(app_name, env_name, os.path.join(config.get_source_dir(), app_name))
+      apps.append(Application(app_name, env_name, workflow))
 
+  log.info('Processing applications')
   try:
-    log.info('Generating temporary files')
-    await asyncio.gather(*[asyncio.create_task(app.prepare()) for app in apps])
-
-    log.info('Rendering resources')
-    await asyncio.gather(*[asyncio.create_task(app.generate_resources()) for app in apps])
+    await asyncio.gather(*[asyncio.create_task(app.process()) for app in apps])
   except Exception:
     for task in asyncio.all_tasks():
       task.cancel()
     raise
 
-  output_writer = ResourceWriter(config.get_output_dir())
-  for app in apps:
-    for resource_kind, resource_name, resource_yml in multi_resource_parser(app.resources):
-      file_path = os.path.join(app.get_app_rel_path(), generate_filename([resource_kind, resource_name]))
-      output_writer.store_resource(file_path, resource_yml)
-
-  if apps:
-    log.info('The following applications have been updated:')
-    for app in apps:
-      app_dir = os.path.join(config.get_output_dir(), app.get_app_rel_path())
-      log.info('Environment: {}, Application: {}, Path: {}'.format(app.env_name, app.app_name, app_dir))
-      if os.path.exists(app_dir):
-        shutil.rmtree(app_dir)
-
-  log.info('Writing resources files')
-  os.makedirs(config.get_output_dir(), exist_ok=True)
-  await output_writer.write_resources()
-
 
 def main() -> None:
   parser = argparse.ArgumentParser(description='Render ArgoCD Applications.')

make_argocd_fly/renderer.py

@@ -13,15 +13,15 @@
 
 class AbstractRenderer(ABC):
   @abstractmethod
-  def render(self, content: str, template_vars: dict = None) -> str:
+  def render(self, content: str) -> str:
     pass
 
 
 class DummyRenderer(AbstractRenderer):
   def __init__(self) -> None:
     pass
 
-  def render(self, content: str, template_vars: dict = None) -> str:
+  def render(self, content: str) -> str:
     return content
 
 
@@ -63,6 +63,9 @@ def __init__(self, viewer: ResourceViewer = None) -> None:
                                        'jinja2_ansible_filters.AnsibleCoreFiltersExtension'],
                            loader=self.loader, undefined=StrictUndefined)
 
+    self.template_vars = {}
+    self.filename = '<template>'
+
   def _get_template(self, path: str):
     files_children = self.viewer.get_files_children(os.path.basename(path))
     for file_child in files_children:
@@ -72,8 +75,14 @@ def _get_template(self, path: str):
     log.error('Missing template {}'.format(path))
     return None
 
-  def render(self, content: str, template_vars: dict = None, filename: str = '<template>') -> str:
+  def set_template_vars(self, template_vars: dict) -> None:
+    self.template_vars = template_vars
+
+  def set_filename(self, filename: str) -> None:
+    self.filename = filename
+
+  def render(self, content: str) -> str:
     template = self.env.from_string(content)
-    template.filename = filename
+    template.filename = self.filename
 
-    return template.render(template_vars if template_vars else {})
+    return template.render(self.template_vars)

make_argocd_fly/resource.py

@@ -3,11 +3,14 @@
 import re
 import asyncio
 import yaml
+import yaml.composer
 try:
   from yaml import CSafeLoader as SafeLoader
 except ImportError:
   from yaml import SafeLoader
 
+from make_argocd_fly.exceptions import MissingSourceResourcesError
+
 log = logging.getLogger(__name__)
 
 
@@ -29,10 +32,6 @@ def str_presenter(dumper, data):
 
 class ResourceViewer:
   def __init__(self, root_element_abs_path: str, element_rel_path: str = '.', is_dir: bool = True) -> None:
-    if not os.path.exists(root_element_abs_path):
-      log.error('Path does not exist {}'.format(root_element_abs_path))
-      raise Exception
-
     self.root_element_abs_path = root_element_abs_path
     self.element_rel_path = os.path.normpath(element_rel_path)
     self.is_dir = is_dir
@@ -46,6 +45,9 @@ def __init__(self, root_element_abs_path: str, element_rel_path: str = '.', is_d
     self.children = []
 
   def build(self) -> None:
+    if not os.path.exists(self.root_element_abs_path):
+      raise MissingSourceResourcesError('Path does not exist {}'.format(self.root_element_abs_path))
+
     path = os.path.join(self.root_element_abs_path, self.element_rel_path)
     if not os.path.exists(path):
       log.error('Path does not exist {}'.format(path))
@@ -137,7 +139,13 @@ async def _write_resource(self, file_path: str, resource_yml: str) -> None:
 
     with open(os.path.join(self.output_dir_abs_path, file_path), 'w') as f:
       f.write(resource_yml)
-    yaml_obj = yaml.load(resource_yml, Loader=SafeLoader)
+
+    try:
+      yaml_obj = yaml.load(resource_yml, Loader=SafeLoader)
+    except yaml.composer.ComposerError:
+      log.error('Error parsing yaml to write as file {}. Yaml:\n{}'.format(file_path, resource_yml))
+      raise
+
     with open(os.path.join(self.output_dir_abs_path, file_path), 'w') as f:
       yaml.dump(yaml_obj, f, Dumper=SafeDumper,
                 default_flow_style=False,
@@ -152,4 +160,6 @@ async def write_resources(self) -> None:
         *[asyncio.create_task(self._write_resource(file_path, resource_yml)) for file_path, resource_yml in self.resources.items()]
       )
     except Exception:
+      for task in asyncio.all_tasks():
+        task.cancel()
       raise