From 160aa4c4baf9afa6a1d7badc99eedb2c9bcd2689 Mon Sep 17 00:00:00 2001 From: Aaditya Sharma Date: Wed, 22 Jan 2020 13:26:47 +0530 Subject: [PATCH 01/10] Added readme.md --- README.md | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 338bf77..785063a 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,6 @@ projects and make development + deployment of projects seamless. ## Installation -Clone the repository.
Create a new python virtualenv. You can use the following command.
```bash python3 -m venv @@ -16,11 +15,18 @@ Activate your virtualenv
source /path_to_venv/bin/activate ``` +```bash +pip3 install afctl +``` +or
+ +Clone the repository.
Step into the afctl directory and run
```bash pip3 install . ``` + ## Requirements Python 3.5+ Docker @@ -160,6 +166,13 @@ afctl config show * Prints the configuration file on the console. +### Example +* Qubole +```bash +afctl config add -d qubole -n name -e https://api.qubole.com -c cluster-label -t Auth-token +afctl deploy qubole -n name +``` + ### Caution Not yet ported for Windows. From 544c6ee63b82783568e351591022d16078401e60 Mon Sep 17 00:00:00 2001 From: Aaditya Sharma Date: Thu, 13 Feb 2020 22:11:08 +0530 Subject: [PATCH 02/10] Added version for airflow --- afctl/parser_helpers.py | 3 +++ afctl/parsers.py | 13 ++++++++++--- afctl/plugins/deployments/deployment_config.yml | 1 + afctl/templates/dag_template.py | 2 +- 4 files changed, 15 insertions(+), 4 deletions(-) diff --git a/afctl/parser_helpers.py b/afctl/parser_helpers.py index 000b094..a424125 100644 --- a/afctl/parser_helpers.py +++ b/afctl/parser_helpers.py @@ -113,6 +113,9 @@ def generate_project(args, files): # Since its an afctl project. Just populate the config files. ParserHelpers.generate_config_file(files) + if args.v != None: + Utility.update_config(files['project_name'], {'global': {'airflow_version': args.v}}) + return False except Exception as e: raise AfctlParserException(e) diff --git a/afctl/parsers.py b/afctl/parsers.py index 6868038..8242793 100644 --- a/afctl/parsers.py +++ b/afctl/parsers.py @@ -145,7 +145,8 @@ def get_subparsers(cls): 'parser': 'init', 'help': 'Create a new Airflow project.', 'args': [ - ['name', {'help':'Name of your airflow project'}] + ['name', {'help':'Name of your airflow project'}], + ['-v', {'help': 'Airflow version of your project'}] ] }, @@ -185,7 +186,8 @@ def get_subparsers(cls): ['-n'], ['-e'], ['-c'], - ['-t'] + ['-t'], + ['-v'] ] }, @@ -262,9 +264,11 @@ def act_on_configs(cls, args, config_file): if args.type == "global": origin = args.o token = args.t - if args.o is None and args.t is None: + version = args.v + if args.o is None and args.t is None and args.v is None: origin = input("Git origin for deployment : ") token = input("Personal access token : ") + version = input("Input airflow version : ") if origin != '' and origin is not None: Utility.update_config(config_file, {'global':{'git':{'origin':origin}}}) @@ -272,6 +276,9 @@ def act_on_configs(cls, args, config_file): if token != '' and token is not None: Utility.update_config(config_file, {'global':{'git':{'access-token':token}}}) + if version != '' and version is not None: + Utility.update_config(config_file, {'global': {'airflow_version': version}}) + # If adding or updating configs. elif args.type == 'add' or args.type == 'update': if args.d is None: diff --git a/afctl/plugins/deployments/deployment_config.yml b/afctl/plugins/deployments/deployment_config.yml index 0ae45f7..9f7be15 100644 --- a/afctl/plugins/deployments/deployment_config.yml +++ b/afctl/plugins/deployments/deployment_config.yml @@ -1,4 +1,5 @@ global: + airflow_version: git: origin: access-token: diff --git a/afctl/templates/dag_template.py b/afctl/templates/dag_template.py index ad16933..cca00fa 100644 --- a/afctl/templates/dag_template.py +++ b/afctl/templates/dag_template.py @@ -17,7 +17,7 @@ def dag_template(name, config_name): } -dag = DAG(dag_id=${name}, default_args=default_args, schedule_interval='@once') +dag = DAG(dag_id='${name}', default_args=default_args, schedule_interval='@once') """ From c7c09bc5aa8d8a79bbe706c427fd1027d1b5a25b Mon Sep 17 00:00:00 2001 From: Aaditya Sharma Date: Tue, 18 Feb 2020 11:32:30 +0530 Subject: [PATCH 03/10] Added requirements.txt --- afctl/parser_helpers.py | 2 +- afctl/plugins/deployments/docker/afctl_docker_compose.py | 3 +++ afctl/utils.py | 5 ++++- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/afctl/parser_helpers.py b/afctl/parser_helpers.py index a424125..43ec20a 100644 --- a/afctl/parser_helpers.py +++ b/afctl/parser_helpers.py @@ -15,7 +15,7 @@ def init_file_name(name): project_name = os.path.basename(main_dir) config_dir = Utility.CONSTS['config_dir'] config_file = Utility.project_config(project_name) - sub_files = ['.afctl_project', '.gitignore'] + sub_files = ['.afctl_project', '.gitignore', 'requirements.txt'] sub_dirs = [project_name, 'deployments', 'migrations', 'plugins', 'tests'] project_dirs = ['dags', 'commons'] diff --git a/afctl/plugins/deployments/docker/afctl_docker_compose.py b/afctl/plugins/deployments/docker/afctl_docker_compose.py index 4399295..4e11419 100644 --- a/afctl/plugins/deployments/docker/afctl_docker_compose.py +++ b/afctl/plugins/deployments/docker/afctl_docker_compose.py @@ -37,6 +37,7 @@ def docker_compose_template(path): # - REDIS_PASSWORD=redispass volumes: - ../${path}/dags:/usr/local/airflow/dags + - ../requirements.txt:/requirements.txt # Uncomment to include custom plugins # - ../plugins:/usr/local/airflow/plugins ports: @@ -67,6 +68,7 @@ def docker_compose_template(path): - webserver volumes: - ../${path}/dags:/usr/local/airflow/dags + - ../requirements.txt:/requirements.txt # Uncomment to include custom plugins # - ../plugins:/usr/local/airflow/plugins environment: @@ -86,6 +88,7 @@ def docker_compose_template(path): - scheduler volumes: - ../${path}/dags:/usr/local/airflow/dags + - ../requirements.txt:/requirements.txt # Uncomment to include custom plugins # - ../plugins:/usr/local/airflow/plugins environment: diff --git a/afctl/utils.py b/afctl/utils.py index cc1c7fb..fe98b03 100644 --- a/afctl/utils.py +++ b/afctl/utils.py @@ -32,7 +32,10 @@ def create_files(parent, child): try: files = {} for dir1, dir2 in itertools.product(parent, child): - subprocess.run(['touch', os.path.join(dir1, dir2)]) + if not os.path.exists(os.path.join(dir1, dir2)): + subprocess.run(['touch', os.path.join(dir1, dir2)]) + else: + print("{} already exists. Skipping.".format(dir2)) files[dir2] = os.path.join(dir1, dir2) return files except Exception as e: From 7628c26f64fa90b816bd9fffc332d0ee13517e9a Mon Sep 17 00:00:00 2001 From: Aaditya Sharma Date: Mon, 2 Mar 2020 21:46:31 +0530 Subject: [PATCH 04/10] Added documentation --- README.md | 196 +++++++++++++++++++++++++++++------------------ afctl/parsers.py | 2 +- 2 files changed, 122 insertions(+), 76 deletions(-) diff --git a/README.md b/README.md index 957d693..e9c77f5 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # afctl -The proposed CLI tool is being authored to make creating and deployment of airflow projects faster and smoother. +The proposed CLI tool is authored to make creating and deployment of airflow projects faster and smoother. As of now, there is no tool out there that can empower the user to create a boilerplate code structure for airflow projects and make development + deployment of projects seamless. @@ -20,16 +20,16 @@ pip3 install afctl ``` or
-Clone the repository.
-Step into the afctl directory and run
+Clone this repository.
+Move into the repository and run
```bash pip3 install . ``` ## Requirements -Python 3.5+ -Docker +* Python 3.5+ +* Docker ## Usage @@ -46,7 +46,7 @@ afctl -h ```
-#### Initialize a new afctl project. +## Initialize a new afctl project. The project is created in your present working directory. Along with this a configuration file with the same name is generated in **/home/.afctl_configs** directory. @@ -58,20 +58,23 @@ afctl init “name of the project” * Creates a new project directory. * Creates a config file in the home directory +If you already have a git repository and want to turn it into an afctl project. +Run the following command :- ```bash afctl init . ``` * Initialize the current directory as a project -* If the directory is already a git repository then the global configs will get automatically set.
-#### Manage configurations +## Manage configurations -The configuration file is used for deployment. +The configuration file is used for deployment contains the following information. ```yaml global: +-airflow_version: -git: --origin: +--access-token: deployment: -qubole: --local: @@ -79,105 +82,148 @@ deployment: ```
+* `airflow_version` can be added to the project when you initialize the project. +```bash +afctl init -v ``` -TYPES: - add - add a config for your deployment. - update - update an existing config for your deployment. - Arguments: - -d : Deployment Type - -p : Project - [ Qubole ] - -n : name of deployment - -e : name of environment - -c : cluster label - -t : auth token - [ docker-local ] - Cannot add/update configs. - global - Arguments: - -p : Project - -o : Set git origin for deployment - -t : Set personal access token - show - Show the config file on console - No arguments. -positional arguments: - {add,update,show,global} +* `global configs (airflow_version, origin, access-token)` can all be added/ updated with the following command : + +```bash +afctl config global -o -t -v +``` -optional arguments: - -h, --help show this help message and exit - -d {qubole} - -o O - -p P - -n N - -e E - -c C - -t T +## Deploy project locally +* Initialize a new afctl project +```bash +afctl init project_demo ``` +* The following directory structure will be generated +```bash +. +├── deployments +│   └── project_demo-docker-compose.yml +├── migrations +├── plugins +├── project_demo +│   ├── commons +│   └── dags +├── requirements.txt +└── tests +``` +You can add python packages that will be required by your dags in `requirements.txt`. They will automatically get +installed. +* Add a new module in the project. +```bash +afctl generate module -n +``` + +The following directory structure will be generated : ```bash -afctl config global -o +afctl generate module -n first_module +afctl generate module -n second_module + +. +├── deployments +│   └── project_demo-docker-compose.yml +├── migrations +├── plugins +├── project_demo +│   ├── commons +│   └── dags +│   ├── first_module +│   └── second_module +├── requirements.txt +└── tests + ├── first_module + └── second_module + ``` -* Will set the git origin for your project. -* Supports both inline arguments as well as promoting for input. + +* You can generate dags using the following command : ```bash -afctl configs add -d +afctl generate dag -n -m ``` -* Prompts the user to input connector related values. -* Can also provide values as inline arguments. + +The following directory structure will be generate : ```bash -afctl configs update -d +afctl generate dag -n new -m first_module + +. +├── deployments +│   └── project_demo-docker-compose.yml +├── migrations +├── plugins +├── project_demo +│   ├── commons +│   └── dags +│   ├── first_module +│   │   └── new_dag.py +│   └── second_module +├── requirements.txt +└── tests + ├── first_module + └── second_module ``` -* Prompts the user to update the values -* Can also provide inline arguments. +The dag file will look like this : -#### Deploy projects +```python +from airflow import DAG +from datetime import datetime, timedelta -```bash -TYPES: - [qubole] - Deploy your project to QDS. - Arguments: - -n : Name of the deployment - [local] - Deploy your project to local docker. - Arguments: - -d : To run in daemon mode +default_args = { +'owner': 'project_demo', +# 'depends_on_past': , +# 'start_date': , +# 'email': , +# 'email_on_failure': , +# 'email_on_retry': , +# 'retries': 0 -positional arguments: - {qubole,local} +} -optional arguments: - -h, --help show this help message and exit - -d - -n N +dag = DAG(dag_id='new', default_args=default_args, schedule_interval='@once') ``` -* To Deploy on local docker +* To deploy your project, run the following command (make sure docker is running) : + ```bash afctl deploy local ``` +If you do not want to see the logs, you can run +```bash +afctl deploy local -d +``` +This will run it in detached mode and won't print the logs on the console. + +* You can access your airflow webserver on browser at `localhost:8080` -* To deploy on remote machine +## Deploy project on Qubole +* Initialize a new project and add git-origin and access-token (if want to keep the project as private repo +on Github) to the configs. +* Push the project once completed to Github. +* Deploying to Qubole will require adding deployment configurations. ```bash -afctl deploy -n +afctl config add -d qubole -n -e -c -t ``` -* Fetches the latest commit in remote origin. -* Deploys the repository with the last commit as shown. - +This command will modify your config file. You can see your config file with the following command : ```bash afctl config show ``` -* Prints the configuration file on the console. +For example - +```bash +afctl config add -d qubole -n demo -e https://api.qubole.com -c airflow_1102 -t khd34djs3 +``` -### Example -* Qubole +* To deploy run the following command ```bash -afctl config add -d qubole -n -e https://.qubole.com -c -t -afctl deploy qubole -n name +afctl deploy qubole -n ``` ### Caution diff --git a/afctl/parsers.py b/afctl/parsers.py index 8242793..376ab9f 100644 --- a/afctl/parsers.py +++ b/afctl/parsers.py @@ -146,7 +146,7 @@ def get_subparsers(cls): 'help': 'Create a new Airflow project.', 'args': [ ['name', {'help':'Name of your airflow project'}], - ['-v', {'help': 'Airflow version of your project'}] + ['-v', {'help': 'Airflow version for your project'}] ] }, From 9a33d2c9fbce348e3a8f4c5b84ecd3f67b515fd2 Mon Sep 17 00:00:00 2001 From: Aaditya Sharma Date: Wed, 4 Mar 2020 10:54:21 +0530 Subject: [PATCH 05/10] Changes as per comments --- README.md | 108 ++++++++++++++++++++++++++---------------------------- 1 file changed, 51 insertions(+), 57 deletions(-) diff --git a/README.md b/README.md index e9c77f5..3558ac1 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,10 @@ The proposed CLI tool is authored to make creating and deployment of airflow pro As of now, there is no tool out there that can empower the user to create a boilerplate code structure for airflow projects and make development + deployment of projects seamless. +## Requirements +* Python 3.5+ +* Docker + ## Installation Create a new python virtualenv. You can use the following command.
@@ -18,33 +22,6 @@ source /path_to_venv/bin/activate ```bash pip3 install afctl ``` -or
- -Clone this repository.
-Move into the repository and run
-```bash -pip3 install . -``` - - -## Requirements -* Python 3.5+ -* Docker - -## Usage - -Commands right now supported are -* init -* config -* deploy -* list -* generate - -To learn more, run -```bash -afctl -h -``` -
## Initialize a new afctl project. The project is created in your present working directory. Along with this a configuration file with the same name is @@ -52,7 +29,7 @@ generated in **/home/.afctl_configs** directory. ```bash -afctl init “name of the project” +afctl init ``` * Creates a new project directory. @@ -66,33 +43,6 @@ afctl init . * Initialize the current directory as a project
-## Manage configurations - -The configuration file is used for deployment contains the following information. -```yaml -global: --airflow_version: --git: ---origin: ---access-token: -deployment: --qubole: ---local: ----compose: -``` -
- -* `airflow_version` can be added to the project when you initialize the project. -```bash -afctl init -v -``` - -* `global configs (airflow_version, origin, access-token)` can all be added/ updated with the following command : - -```bash -afctl config global -o -t -v -``` - ## Deploy project locally * Initialize a new afctl project ```bash @@ -202,12 +152,14 @@ This will run it in detached mode and won't print the logs on the console. * You can access your airflow webserver on browser at `localhost:8080` -## Deploy project on Qubole +## Deploy project on production +* Here we will be deploying our project to Qubole. * Initialize a new project and add git-origin and access-token (if want to keep the project as private repo -on Github) to the configs. +on Github) to the configs. [See how](#manage-configurations) * Push the project once completed to Github. * Deploying to Qubole will require adding deployment configurations. + ```bash afctl config add -d qubole -n -e -c -t ``` @@ -226,6 +178,48 @@ afctl config add -d qubole -n demo -e https://api.qubole.com -c airflow_1102 -t afctl deploy qubole -n ``` +## Manage configurations + +The configuration file is used for deployment contains the following information. +```yaml +global: +-airflow_version: +-git: +--origin: +--access-token: +deployment: +-qubole: +--local: +---compose: +``` +
+ +* `airflow_version` can be added to the project when you initialize the project. +```bash +afctl init -v +``` + +* `global configs (airflow_version, origin, access-token)` can all be added/ updated with the following command : + +```bash +afctl config global -o -t -v +``` + +## Usage + +Commands right now supported are +* init +* config +* deploy +* list +* generate + +To learn more, run +```bash +afctl -h +``` +
+ ### Caution Not yet ported for Windows. From 84af4f27ec353e614b0b644abe21ae87f1f4f1f1 Mon Sep 17 00:00:00 2001 From: Manzur Husain Date: Fri, 6 Mar 2020 13:06:17 +0530 Subject: [PATCH 06/10] Update README.md --- README.md | 46 +++++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 3558ac1..7a74917 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,8 @@ projects and make development + deployment of projects seamless. * Python 3.5+ * Docker -## Installation +## Getting Started +### 1. Installation Create a new python virtualenv. You can use the following command.
```bash @@ -23,7 +24,7 @@ source /path_to_venv/bin/activate pip3 install afctl ``` -## Initialize a new afctl project. +### 2. Initialize a new afctl project. The project is created in your present working directory. Along with this a configuration file with the same name is generated in **/home/.afctl_configs** directory. @@ -31,20 +32,7 @@ generated in **/home/.afctl_configs** directory. ```bash afctl init ``` - -* Creates a new project directory. -* Creates a config file in the home directory - -If you already have a git repository and want to turn it into an afctl project. -Run the following command :- -```bash -afctl init . -``` -* Initialize the current directory as a project -
- -## Deploy project locally -* Initialize a new afctl project +Eg. ```bash afctl init project_demo ``` @@ -61,9 +49,16 @@ afctl init project_demo ├── requirements.txt └── tests ``` -You can add python packages that will be required by your dags in `requirements.txt`. They will automatically get -installed. -* Add a new module in the project. + +If you already have a git repository and want to turn it into an afctl project. +Run the following command :- +```bash +afctl init . +``` +
+ +### 3. Add a new module in the project. + ```bash afctl generate module -n ``` @@ -91,8 +86,7 @@ afctl generate module -n second_module ``` -* You can generate dags using the following command : - +### 4. Generate dag ```bash afctl generate dag -n -m ``` @@ -139,11 +133,17 @@ default_args = { dag = DAG(dag_id='new', default_args=default_args, schedule_interval='@once') ``` +### 3. Deploy project locally + +You can add python packages that will be required by your dags in `requirements.txt`. They will automatically get +installed. + * To deploy your project, run the following command (make sure docker is running) : ```bash afctl deploy local ``` + If you do not want to see the logs, you can run ```bash afctl deploy local -d @@ -154,8 +154,8 @@ This will run it in detached mode and won't print the logs on the console. ## Deploy project on production -* Here we will be deploying our project to Qubole. -* Initialize a new project and add git-origin and access-token (if want to keep the project as private repo +* Here we will be deploying our project to **Qubole**. Sign up at us.qubole.com. +* add git-origin and access-token (if want to keep the project as private repo on Github) to the configs. [See how](#manage-configurations) * Push the project once completed to Github. * Deploying to Qubole will require adding deployment configurations. From 8028496401d349d49404a21e19a8ac0559f989e7 Mon Sep 17 00:00:00 2001 From: Manzur Husain Date: Fri, 6 Mar 2020 13:07:46 +0530 Subject: [PATCH 07/10] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 7a74917..0e32d54 100644 --- a/README.md +++ b/README.md @@ -133,7 +133,7 @@ default_args = { dag = DAG(dag_id='new', default_args=default_args, schedule_interval='@once') ``` -### 3. Deploy project locally +### 5. Deploy project locally You can add python packages that will be required by your dags in `requirements.txt`. They will automatically get installed. @@ -152,7 +152,7 @@ This will run it in detached mode and won't print the logs on the console. * You can access your airflow webserver on browser at `localhost:8080` -## Deploy project on production +### 6. Deploy project on production * Here we will be deploying our project to **Qubole**. Sign up at us.qubole.com. * add git-origin and access-token (if want to keep the project as private repo From df1d3373d7e6e4d5d3012c3a3e82a24ad098ad52 Mon Sep 17 00:00:00 2001 From: Manzur Husain Date: Fri, 6 Mar 2020 13:23:30 +0530 Subject: [PATCH 08/10] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0e32d54..bf73026 100644 --- a/README.md +++ b/README.md @@ -152,7 +152,7 @@ This will run it in detached mode and won't print the logs on the console. * You can access your airflow webserver on browser at `localhost:8080` -### 6. Deploy project on production +### 6. Deploy project on production * Here we will be deploying our project to **Qubole**. Sign up at us.qubole.com. * add git-origin and access-token (if want to keep the project as private repo @@ -199,7 +199,7 @@ deployment: afctl init -v ``` -* `global configs (airflow_version, origin, access-token)` can all be added/ updated with the following command : +* global configs (airflow_version, origin, access-token) can all be added/ updated with the following command : ```bash afctl config global -o -t -v From f422b07022deb9b0ac6f29dc9b88d2fe306d9968 Mon Sep 17 00:00:00 2001 From: Manzur Husain Date: Fri, 6 Mar 2020 13:41:45 +0530 Subject: [PATCH 09/10] Update README.md --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index bf73026..bf46110 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,9 @@ projects and make development + deployment of projects seamless. * Docker ## Getting Started +### The following video also contains all the steps of deploying project using afctl -
+https://www.youtube.com/watch?v=A4rcZDGtJME&feature=youtu.be + ### 1. Installation Create a new python virtualenv. You can use the following command.
@@ -24,6 +27,7 @@ source /path_to_venv/bin/activate pip3 install afctl ``` + ### 2. Initialize a new afctl project. The project is created in your present working directory. Along with this a configuration file with the same name is generated in **/home/.afctl_configs** directory. From e79afec430ded04f8852520fb718405622d9a7c7 Mon Sep 17 00:00:00 2001 From: Aaditya Sharma Date: Sun, 8 Mar 2020 16:03:59 +0530 Subject: [PATCH 10/10] minor fix --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index bf46110..ec08e53 100644 --- a/README.md +++ b/README.md @@ -9,8 +9,6 @@ projects and make development + deployment of projects seamless. * Docker ## Getting Started -### The following video also contains all the steps of deploying project using afctl -
-https://www.youtube.com/watch?v=A4rcZDGtJME&feature=youtu.be ### 1. Installation @@ -182,6 +180,9 @@ afctl config add -d qubole -n demo -e https://api.qubole.com -c airflow_1102 -t afctl deploy qubole -n ``` +### The following video also contains all the steps of deploying project using afctl -
+https://www.youtube.com/watch?v=A4rcZDGtJME&feature=youtu.be + ## Manage configurations The configuration file is used for deployment contains the following information.