From 4a1912a9fb38d54e1b02af54447599c2578e1e4f Mon Sep 17 00:00:00 2001 From: Esteban Beltran Date: Wed, 14 Jul 2021 11:41:14 +0200 Subject: [PATCH 1/5] Add a command to run elastic search --- src/commands/check.py | 13 +++++----- src/commands/es.py | 53 ++++++++++++++++++++++++++++++++++++++ src/commands/functional.py | 11 -------- src/config.py | 16 ++++++++++++ src/main.py | 9 ++++--- src/util.py | 6 ++--- 6 files changed, 84 insertions(+), 24 deletions(-) create mode 100644 src/commands/es.py delete mode 100644 src/commands/functional.py create mode 100644 src/config.py diff --git a/src/commands/check.py b/src/commands/check.py index 2a8ad6c..9550020 100644 --- a/src/commands/check.py +++ b/src/commands/check.py @@ -1,12 +1,13 @@ -from pathlib import PurePath +import os import subprocess -from typing import DefaultDict +from pathlib import PurePath + import click -import os from termcolor import colored -import termcolor - -from src.util import find_related_plugin_folder, find_related_test, force_kibana_root, get_modified_files +from click.types import Path +from src.util import (find_related_plugin_folder, find_related_test, + force_kibana_root, get_modified_files) +from termcolor import colored @click.command(help="""Run quick checks for your modified files. Useful to run before committing""") diff --git a/src/commands/es.py b/src/commands/es.py new file mode 100644 index 0000000..f5239ca --- /dev/null +++ b/src/commands/es.py @@ -0,0 +1,53 @@ +import re +import subprocess + +from termcolor import colored +from src.config import get_config +import click +import tempfile +from pathlib import Path, PurePath + +pathDataRe = re.compile(r"path\.data\s?=", re.IGNORECASE) + + +@click.command(help="Runs elastic search from the current kibana clone") +@click.option('--data-dir', '-d', type=click.STRING, default="esdata", help="Where this elastic search will store its data") +@click.option('--no-persist', '-n', default=False, is_flag=True, help="If True will use a disposable data dir. This option will overwrite other options related to data dir.") +@click.option('-E', multiple=True, help="Additional options to pass to elastic search. `path.data` will be ignored") +def es(data_dir, no_persist, e): + CONFIG_KEY = 'elastic.params' + config = get_config() + params = [] + if CONFIG_KEY in config: + for (key, value) in config.items(CONFIG_KEY, raw=True): + # ignore path.data if this command overwrites it + if key == 'path.data': + if len(data_dir) > 0: + value = get_data_dir(data_dir, no_persist) + else: + value = get_data_dir(value, no_persist) + + params.append(str(key) + '=' + str(value)) + + for item in e: + item = item.strip() + # ignore path.data + if pathDataRe.match(item): + continue + params.append(item) + + final_params = [] + for param in params: + final_params.append('-E') + final_params.append(param) + + command = ['node', 'scripts/es', 'snapshot'] + final_params + click.echo("Will run elastic search as: " + colored(' '.join(command), 'yellow')) + # subprocess.run(command) + + +def get_data_dir(data_dir, no_persist): + if no_persist or len(data_dir) == 0: + return tempfile.mkdtemp(suffix='kibbe') + + return str(Path(data_dir).resolve()) diff --git a/src/commands/functional.py b/src/commands/functional.py deleted file mode 100644 index f1bd61f..0000000 --- a/src/commands/functional.py +++ /dev/null @@ -1,11 +0,0 @@ -import click - - -@click.group(help="""Utilities for functional tests""") -def functional(): - pass - - -@functional.command(help="""List the available projects to run functional tests""") -def list(): - print("im listing stuff") diff --git a/src/config.py b/src/config.py new file mode 100644 index 0000000..3b2e525 --- /dev/null +++ b/src/config.py @@ -0,0 +1,16 @@ +from pathlib import Path, PurePath +import configparser +from termcolor import colored + +import click + + +def get_config(): + config = configparser.ConfigParser() + try: + config_file_path = PurePath(Path.home()).joinpath('.kibbe') + config.read(config_file_path) + except ValueError: + click.echo(colored('Error reading your configuration file.', 'red')) + pass + return config diff --git a/src/main.py b/src/main.py index d8a25d8..0de449b 100644 --- a/src/main.py +++ b/src/main.py @@ -1,8 +1,9 @@ -from src.util import force_kibana_root import click -from src.commands.setlicense import setlicense -from src.commands.functional import functional + from src.commands.check import check +from src.commands.es import es +from src.commands.setlicense import setlicense +from src.util import force_kibana_root @click.group() @@ -11,5 +12,5 @@ def cli(): cli.add_command(setlicense) -cli.add_command(functional) cli.add_command(check) +cli.add_command(es) diff --git a/src/util.py b/src/util.py index 049f928..08de536 100644 --- a/src/util.py +++ b/src/util.py @@ -1,10 +1,10 @@ -import os import json +import os import subprocess -from typing import ValuesView -import click from pathlib import PurePath +import click + def is_tool(name): """Check whether `name` is on PATH and marked as executable.""" From 300ec7d51bcb6857e4901ddb13cf574841d4c37c Mon Sep 17 00:00:00 2001 From: Esteban Beltran Date: Wed, 14 Jul 2021 11:45:59 +0200 Subject: [PATCH 2/5] Improve options message --- kibbe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kibbe.py b/kibbe.py index 83e3021..80d103a 100755 --- a/kibbe.py +++ b/kibbe.py @@ -1,3 +1,3 @@ #!/usr/bin/env python3 from src import main -main.cli() +main.sdfsdfd() From 0754f1887d21c2663b29547a779cd1d8a9716df1 Mon Sep 17 00:00:00 2001 From: Esteban Beltran Date: Wed, 14 Jul 2021 11:48:25 +0200 Subject: [PATCH 3/5] Fix main file error --- kibbe.py | 2 +- src/commands/es.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/kibbe.py b/kibbe.py index 80d103a..83e3021 100755 --- a/kibbe.py +++ b/kibbe.py @@ -1,3 +1,3 @@ #!/usr/bin/env python3 from src import main -main.sdfsdfd() +main.cli() diff --git a/src/commands/es.py b/src/commands/es.py index f5239ca..2c8055a 100644 --- a/src/commands/es.py +++ b/src/commands/es.py @@ -10,9 +10,9 @@ pathDataRe = re.compile(r"path\.data\s?=", re.IGNORECASE) -@click.command(help="Runs elastic search from the current kibana clone") -@click.option('--data-dir', '-d', type=click.STRING, default="esdata", help="Where this elastic search will store its data") -@click.option('--no-persist', '-n', default=False, is_flag=True, help="If True will use a disposable data dir. This option will overwrite other options related to data dir.") +@click.command(help="Runs elastic search from the current kibana clone. It will use parameters from the ~/.kibbe [elastic.params] section.") +@click.option('--data-dir', '-d', type=click.STRING, default="esdata", help="Path where this elastic search will store its data (path.data)") +@click.option('--no-persist', '-n', default=False, is_flag=True, help="If passed will use a disposable data dir. This option will overwrite other options related to data dir.") @click.option('-E', multiple=True, help="Additional options to pass to elastic search. `path.data` will be ignored") def es(data_dir, no_persist, e): CONFIG_KEY = 'elastic.params' From a9c360df6f7d6ca700dcf12cb709672bd0cb9c08 Mon Sep 17 00:00:00 2001 From: Esteban Beltran Date: Wed, 14 Jul 2021 12:00:07 +0200 Subject: [PATCH 4/5] Add a configuration example. Allow flags in configuration file --- kibbe-conf-example | 13 +++++++++++++ src/commands/es.py | 5 ++++- src/main.py | 11 ++++++++++- 3 files changed, 27 insertions(+), 2 deletions(-) create mode 100644 kibbe-conf-example diff --git a/kibbe-conf-example b/kibbe-conf-example new file mode 100644 index 0000000..60009d1 --- /dev/null +++ b/kibbe-conf-example @@ -0,0 +1,13 @@ +# This is kibbe configuration file example + +# Each parameter here will be passed to elastic search as an `-E` parameter. +# Note that parameters without value still require an equal sign `=` at the end +# the sign will be ignored when invoking +[elastic.params] +xpack.security.authc.api_key.enabled=true + +# Each parameter here will be passed to kibana when run +# Note that parameters without value still require an equal sign `=` at the end +# the sign will be ignored when invoking +[kibana.params] +--no-base-path= \ No newline at end of file diff --git a/src/commands/es.py b/src/commands/es.py index 2c8055a..5173507 100644 --- a/src/commands/es.py +++ b/src/commands/es.py @@ -27,7 +27,10 @@ def es(data_dir, no_persist, e): else: value = get_data_dir(value, no_persist) - params.append(str(key) + '=' + str(value)) + if len(value) > 0: + params.append(str(key) + '=' + str(value)) + else: + params.append(str(key)) for item in e: item = item.strip() diff --git a/src/main.py b/src/main.py index 0de449b..db78c74 100644 --- a/src/main.py +++ b/src/main.py @@ -6,7 +6,16 @@ from src.util import force_kibana_root -@click.group() +@click.group(help=""" + Kibbe is a tool that help with common tasks when developing kibana plugins. + + Some subcommands allow you to define a configuration to persist arguments for some + specific tasks such as running kibana or elasticsearch. This configuration file will + persis those arguments among all kibana clones and branches. + + You can create a configuration file in your home ~/.kibbe and follow the configuration + example in the kibbe repository +""") def cli(): force_kibana_root() From bb8413c30d90fe4d707e1230f94566aed2185314 Mon Sep 17 00:00:00 2001 From: Esteban Beltran Date: Wed, 14 Jul 2021 12:05:42 +0200 Subject: [PATCH 5/5] Improve code structure --- src/commands/es.py | 44 ++++++++++++++++++++++++++++---------------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/src/commands/es.py b/src/commands/es.py index 5173507..2f3ae18 100644 --- a/src/commands/es.py +++ b/src/commands/es.py @@ -5,7 +5,7 @@ from src.config import get_config import click import tempfile -from pathlib import Path, PurePath +from pathlib import Path pathDataRe = re.compile(r"path\.data\s?=", re.IGNORECASE) @@ -15,6 +15,32 @@ @click.option('--no-persist', '-n', default=False, is_flag=True, help="If passed will use a disposable data dir. This option will overwrite other options related to data dir.") @click.option('-E', multiple=True, help="Additional options to pass to elastic search. `path.data` will be ignored") def es(data_dir, no_persist, e): + + params = process_params(data_dir, no_persist) + + # additional -E params + for item in e: + item = item.strip() + # ignore path.data + if pathDataRe.match(item): + continue + params.append(item) + + command = get_command(params) + click.echo("Will run elastic search as: " + colored(' '.join(command), 'yellow')) + subprocess.run(command) + + +def get_command(params): + final_params = [] + for param in params: + final_params.append('-E') + final_params.append(param) + + return ['node', 'scripts/es', 'snapshot'] + final_params + + +def process_params(data_dir, no_persist): CONFIG_KEY = 'elastic.params' config = get_config() params = [] @@ -32,21 +58,7 @@ def es(data_dir, no_persist, e): else: params.append(str(key)) - for item in e: - item = item.strip() - # ignore path.data - if pathDataRe.match(item): - continue - params.append(item) - - final_params = [] - for param in params: - final_params.append('-E') - final_params.append(param) - - command = ['node', 'scripts/es', 'snapshot'] + final_params - click.echo("Will run elastic search as: " + colored(' '.join(command), 'yellow')) - # subprocess.run(command) + return params def get_data_dir(data_dir, no_persist):