Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Feature] Improve base branch selection in compare #880

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion piperider_cli/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -444,7 +444,7 @@ def cloud_compare_reports(**kwargs):
@add_options([
dbt_select_option_builder(),
click.option('--modified', default=False, is_flag=True, help='Only compare the modified models.'),
click.option('--base-branch', default='main', type=click.STRING, help='Specify the base branch.',
click.option('--base-branch', type=click.STRING, help='Specify the base branch for an auto-generated recipe.',
show_default=True),
])
@add_options(dbt_related_options)
Expand Down
11 changes: 6 additions & 5 deletions piperider_cli/recipe_executor.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,25 +13,26 @@
class RecipeExecutor:
@staticmethod
def exec(recipe_name: str, auto_generate_default_recipe: bool = True, select: tuple = None, modified: bool = False,
debug=False, base_branch: str = None, target_branch: str = None):
debug=False, base_branch: str = None):
config = Configuration.instance()
recipe_path = select_recipe_file(recipe_name)

if recipe_name and (select or modified is True):
if recipe_name and (select or modified or base_branch):

Check warning on line 20 in piperider_cli/recipe_executor.py

View check run for this annotation

Codecov / codecov/patch

piperider_cli/recipe_executor.py#L20

Added line #L20 was not covered by tests
console.print(
"[[bold yellow]Warning[/bold yellow]] The recipe will be ignored when --select or --modified is provided."
"[[bold yellow]Warning[/bold yellow]] "
"The recipe will be ignored when '--select', '--modified' or '--base-branch' is provided."
)
if select:
console.print(
f"[[bold green]Select[/bold green]] Manually select the dbt nodes to run by '{','.join(select)}'")
if recipe_path is None or select or modified is True:
if recipe_path is None or select or modified or base_branch:

Check warning on line 28 in piperider_cli/recipe_executor.py

View check run for this annotation

Codecov / codecov/patch

piperider_cli/recipe_executor.py#L28

Added line #L28 was not covered by tests
if auto_generate_default_recipe:
dbt_project_path = None
if config.dataSources and config.dataSources[0].args.get('dbt'):
dbt_project_path = os.path.relpath(config.dataSources[0].args.get('dbt', {}).get('projectDir'))
# generate a default recipe
console.rule("Recipe executor: generate recipe")
options = dict(base_branch=base_branch, target_branch=target_branch)
options = dict(base_branch=base_branch)

Check warning on line 35 in piperider_cli/recipe_executor.py

View check run for this annotation

Codecov / codecov/patch

piperider_cli/recipe_executor.py#L35

Added line #L35 was not covered by tests
if select:
options['select'] = select
recipe = generate_default_recipe(overwrite_existing=False,
Expand Down
22 changes: 19 additions & 3 deletions piperider_cli/recipes/default_recipe_generator.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@

from piperider_cli.configuration import FileSystem
from piperider_cli.dbtutil import load_dbt_project
from piperider_cli.error import RecipeException

Check warning on line 9 in piperider_cli/recipes/default_recipe_generator.py

View check run for this annotation

Codecov / codecov/patch

piperider_cli/recipes/default_recipe_generator.py#L9

Added line #L9 was not covered by tests
from piperider_cli.recipes import (RecipeConfiguration,
RecipeDbtField, RecipeModel,
RecipePiperiderField, tool)
Expand All @@ -31,7 +32,22 @@
base = RecipeModel()

if tool().git_branch() is not None:
base.branch = options.get('base_branch', 'main') if options else 'main'
if options.get('base_branch') is not None:
if tool().git_branch(options.get('base_branch')) is not None:
base.branch = options.get('base_branch')

Check warning on line 37 in piperider_cli/recipes/default_recipe_generator.py

View check run for this annotation

Codecov / codecov/patch

piperider_cli/recipes/default_recipe_generator.py#L35-L37

Added lines #L35 - L37 were not covered by tests
else:
ex = RecipeException(f"Cannot find specified base branch: {options.get('base_branch')}")
ex.hint = f"Please check if the specified branch name '{options.get('base_branch')}' is correct."
raise ex

Check warning on line 41 in piperider_cli/recipes/default_recipe_generator.py

View check run for this annotation

Codecov / codecov/patch

piperider_cli/recipes/default_recipe_generator.py#L39-L41

Added lines #L39 - L41 were not covered by tests
else:
if tool().git_branch('main') is not None:
base.branch = 'main'
elif tool().git_branch('master') is not None:
base.branch = 'master'

Check warning on line 46 in piperider_cli/recipes/default_recipe_generator.py

View check run for this annotation

Codecov / codecov/patch

piperider_cli/recipes/default_recipe_generator.py#L43-L46

Added lines #L43 - L46 were not covered by tests
else:
ex = RecipeException("Cannot find default 'main' or 'master' branch")
ex.hint = "Please specify the base branch using the '--base-branch' option."
raise ex

Check warning on line 50 in piperider_cli/recipes/default_recipe_generator.py

View check run for this annotation

Codecov / codecov/patch

piperider_cli/recipes/default_recipe_generator.py#L48-L50

Added lines #L48 - L50 were not covered by tests

dbt_project = _read_dbt_project_file(dbt_project_path)
if dbt_project:
Expand All @@ -48,7 +64,7 @@
return base


def _create_target_recipe(dbt_project_path=None, options: dict = None) -> RecipeModel:
def _create_target_recipe(dbt_project_path=None) -> RecipeModel:

Check warning on line 67 in piperider_cli/recipes/default_recipe_generator.py

View check run for this annotation

Codecov / codecov/patch

piperider_cli/recipes/default_recipe_generator.py#L67

Added line #L67 was not covered by tests
"""
Create the target recipe
"""
Expand Down Expand Up @@ -81,7 +97,7 @@
console.print('[bold green]Piperider default recipe already exist[/bold green]')
return None
base = _create_base_recipe(dbt_project_path, options)
target = _create_target_recipe(dbt_project_path, options)
target = _create_target_recipe(dbt_project_path)

Check warning on line 100 in piperider_cli/recipes/default_recipe_generator.py

View check run for this annotation

Codecov / codecov/patch

piperider_cli/recipes/default_recipe_generator.py#L100

Added line #L100 was not covered by tests
recipe = RecipeConfiguration(base=base, target=target)

try:
Expand Down
7 changes: 4 additions & 3 deletions piperider_cli/recipes/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -88,9 +88,9 @@

return proc.returncode

def git_branch(self):
def git_branch(self, obj_name: str = 'HEAD'):

Check warning on line 91 in piperider_cli/recipes/utils.py

View check run for this annotation

Codecov / codecov/patch

piperider_cli/recipes/utils.py#L91

Added line #L91 was not covered by tests
outs, errs, exit_code = self.dryrun_ignored_execute_command(
"git rev-parse --abbrev-ref HEAD"
f"git rev-parse --abbrev-ref {obj_name}"
)
if exit_code != 0:
return None
Expand All @@ -117,7 +117,8 @@
matched = re.match(r"fatal: Not a valid object name (.*)", errs)
ex.message = f"Invalid git branch: {matched.group(1)}"
ex.hint = (
f"Please check is the branch name '{matched.group(1)}' correct?"
f"Please check if the base branch name '{matched.group(1)}' is correct in your recipe, "
f"or use the '--base-branch' option to specify the branch name for a new auto-generated recipe."
)
raise ex
return outs
Expand Down
Loading