diff --git a/.github/workflows/update_source_data_schema_changelog.yml b/.github/workflows/update_source_data_schema_changelog.yml index 209f6396..0e9188aa 100644 --- a/.github/workflows/update_source_data_schema_changelog.yml +++ b/.github/workflows/update_source_data_schema_changelog.yml @@ -27,12 +27,16 @@ jobs: set -x cd $GITHUB_WORKSPACE OLD_SCHEMAS_DIR=original_schemas + OLD_CHANGELOG_FILEPATH=original_changelog.md git clone --branch master https://github.com/stellar/stellar-etl-airflow.git repo_master_copy mkdir OLD_SCHEMAS_DIR cp -r repo_master_copy/schemas/ $OLD_SCHEMAS_DIR/ + cp repo_master_copy/changelog/source_data.md $OLD_CHANGELOG_FILEPATH export OLD_SCHEMAS_DIR + export OLD_CHANGELOG_FILEPATH python3 scripts/update_source_data_schema_changelog.py rm -rf $OLD_SCHEMAS_DIR + rm $OLD_CHANGELOG_FILEPATH rm -rf repo_master_copy - name: Commit and Push Changes diff --git a/scripts/update_source_data_schema_changelog.py b/scripts/update_source_data_schema_changelog.py index 805cf77e..91da596c 100644 --- a/scripts/update_source_data_schema_changelog.py +++ b/scripts/update_source_data_schema_changelog.py @@ -4,6 +4,7 @@ from datetime import datetime OLD_SCHEMAS_DIR = os.environ.get("OLD_SCHEMAS_DIR") +OLD_CHANGELOG_FILEPATH = os.environ.get("OLD_CHANGELOG_FILEPATH") NEW_SCHEMAS_DIR = "schemas" CHANGELOG_FILEPATH = "changelog/source_data.md" @@ -173,7 +174,7 @@ def generate_changelog(schemas_added=[], schemas_deleted=[], schemas_changes={}) def main(): - existing_changelog = read_file(filepath=CHANGELOG_FILEPATH) + existing_changelog = read_file(filepath=OLD_CHANGELOG_FILEPATH) old_schema_filepaths = list_files_in_dir(directory=OLD_SCHEMAS_DIR) new_schema_filepaths = list_files_in_dir(directory=NEW_SCHEMAS_DIR) @@ -191,6 +192,9 @@ def main(): filepath=CHANGELOG_FILEPATH, mode="w", content=new_changelog + "\n\n" ) write_file(filepath=CHANGELOG_FILEPATH, mode="a", content=existing_changelog) + else: + # Rewrite the changelog since intermediate commits may have overwritten changelog + write_file(filepath=CHANGELOG_FILEPATH, mode="w", content=existing_changelog) if __name__ == "__main__":