From b9319bcf10a396d567515278c377a62828befef9 Mon Sep 17 00:00:00 2001 From: "ivan.ruzavin" Date: Wed, 13 Nov 2024 15:05:12 +0100 Subject: [PATCH 1/5] Initial commit --- scripts/index_to_me.py | 90 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 scripts/index_to_me.py diff --git a/scripts/index_to_me.py b/scripts/index_to_me.py new file mode 100644 index 000000000..fea320b23 --- /dev/null +++ b/scripts/index_to_me.py @@ -0,0 +1,90 @@ +import sys, json, argparse, requests +# from elasticsearch import Elasticsearch + +import classes.class_gh as gh +import classes.class_es as es + +# Skip packages with these types +type_skip = ['microchip_dfp', 'microchip_tp'] + +if __name__ == "__main__": + # First, check for arguments passed + def str2bool(v): + if isinstance(v, bool): + return v + if v.lower() in ('yes', 'true', 't', 'y', '1'): + return True + elif v.lower() in ('no', 'false', 'f', 'n', '0'): + return False + else: + raise argparse.ArgumentTypeError('Boolean value expected.') + + # Get arguments + parser = argparse.ArgumentParser(description="Upload directories as release assets.") + parser.add_argument("gh_repo", help="Github repository name, e.g., 'username/repo'", type=str) + parser.add_argument("gh_token", help="GitHub Token", type=str) + parser.add_argument("es_host", help="ES instance host value", type=str) + parser.add_argument("es_user", help="ES instance user value", type=str) + parser.add_argument("es_password", help="ES instance password value", type=str) + # TODO https://kibana.mikroe.com + parser.add_argument("me_es_host", help="MikroE ES instance host value", type=str) + # TODO sw-github + parser.add_argument("me_es_user", help="MikroE ES instance user value", type=str) + # TODO sw-github334455 + parser.add_argument("me_es_password", help="MikroE ES instance password value", type=str) + parser.add_argument("es_index", help="ES instance index value", type=str) # github_[test|live]_index + parser.add_argument("--es_regex", help="Regex to use to fetch indexed items", type=str, default=".+") + # TODO parser.add_argument("--log_only", help="If True, will not fix broken links, just log them to std out", type=str2bool, default=False) + # TODO parser.add_argument("--index_package_names", help="If True, will add \"gh_package_name\" to indexed item", type=str2bool, default=True) + args = parser.parse_args() + + # es_instance = es.index( + # es_host=args.es_host, es_user=args.es_user, es_password=args.es_password, + # index=args.es_index, token=args.gh_token + # ) + + me_es_instance = es.index( + es_host=args.me_es_host, es_user=args.me_es_user, es_password=args.me_es_password, + index=args.es_index, token=args.gh_token + ) + + gh_instance = gh.repo(args.gh_repo, args.gh_token) + + es_instance.fetch(regex=args.es_regex) + + headers = { + 'Authorization': f'token {args.gh_token}' + } + + err = False + for indexed_item in es_instance.indexed_items: + if indexed_item['source']['type'] in type_skip: + continue + asset_status = requests.get(indexed_item['source']['download_link'], headers=headers) + if es_instance.Status.ERROR.value == asset_status.status_code: ## code 404 - error, reindex with correct download link + err = True + print("%sERROR: Asset \"%s\" download link is incorrect. - %s" % (es_instance.Colors.FAIL, indexed_item['source']['name'], indexed_item['source']['download_link'])) + if 'gh_package_name' in indexed_item['source']: + url = gh_instance.asset_fetch_url_api(indexed_item['source']['gh_package_name'], loose=False) + indexed_item['source']['download_link'] = url + es_instance.update(indexed_item['doc']['type'], indexed_item['doc']['id'], indexed_item['source']) + else: + print("%sWARNING: Asset \"%s\" has no \"gh_package_name\" in the index." % (es_instance.Colors.WARNING, indexed_item['source']['name'])) + else: ## code 200 - success, no need to reindex + package_name = (json.loads(asset_status.text))['name'] + if 'gh_package_name' not in indexed_item['source']: + indexed_item['source'].update({"gh_package_name": package_name}) + es_instance.update(indexed_item['doc']['type'], indexed_item['doc']['id'], indexed_item['source']) + print("%sINFO: Added \"gh_package_name\" to %s" % (es_instance.Colors.UNDERLINE, indexed_item['source']['name'])) + else: + if package_name != indexed_item['source']['gh_package_name']: + indexed_item['source']['gh_package_name'] = package_name + es_instance.update(indexed_item['doc']['type'], indexed_item['doc']['id'], indexed_item['source']) + print("%sINFO: Updated \"gh_package_name\" for %s" % (es_instance.Colors.UNDERLINE, indexed_item['source']['name'])) + print("%sOK: Asset \"%s\" download link is correct. - %s" % (es_instance.Colors.OKBLUE, indexed_item['source']['name'], indexed_item['source']['download_link'])) + + # TODO - index to me_es + me_es_instance.update(indexed_item['doc']['type'], indexed_item['doc']['id'], indexed_item['source']) + + if err and args.log_only: + sys.exit(-1) \ No newline at end of file From 778c18666f0a7e4f893c8b3f32f4d5897609548e Mon Sep 17 00:00:00 2001 From: "ivan.ruzavin" Date: Thu, 14 Nov 2024 12:50:25 +0100 Subject: [PATCH 2/5] Created workflow and tested script --- .github/workflows/copyIndexesToDBP.yaml | 94 +++++++++++++++++++++++++ scripts/classes/class_es.py | 19 +++-- scripts/index_to_me.py | 36 ++-------- 3 files changed, 115 insertions(+), 34 deletions(-) create mode 100644 .github/workflows/copyIndexesToDBP.yaml diff --git a/.github/workflows/copyIndexesToDBP.yaml b/.github/workflows/copyIndexesToDBP.yaml new file mode 100644 index 000000000..6ab0c410b --- /dev/null +++ b/.github/workflows/copyIndexesToDBP.yaml @@ -0,0 +1,94 @@ +name: Copy Indexes from AWS to DBP + +on: + workflow_dispatch: + inputs: + select_index: + type: choice + description: Copy Test and/or Live ES indexed items + options: + - Test + - Live + - Both + regex: + type: string + description: Regex to use when searching for indexed items + default: "arm_gcc_clang|arm_mikroc|clocks|database|dspic|^images$|mikroe_utils|pic|preinit|riscv|schemas|unit_test_lib" + +jobs: + manual_run: + if: ${{ github.event_name == 'workflow_dispatch' }} + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.x' + + - name: Install Dependencies + run: | + python -m pip install --upgrade pip + pip install -r scripts/requirements/check_index.txt + + - name: Copy Indexed Links - Live + if: ${{ github.event.inputs.select_index == 'Live' || github.event.inputs.select_index == 'Both' }} + run: | + python -u scripts/index_to_me.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ME_ES_HOST }} ${{ secrets.ME_ES_USER }} ${{ secrets.ME_ES_PASSWORD }} ${{ secrets.ES_INDEX_LIVE }} "--es_regex" "${{ github.event.inputs.regex }}" + + - name: Copy Indexed Links - Test + if: ${{ github.event.inputs.select_index == 'Test' || github.event.inputs.select_index == 'Both' }} + run: | + python -u scripts/index_to_me.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ME_ES_HOST }} ${{ secrets.ME_ES_USER }} ${{ secrets.ME_ES_PASSWORD }} ${{ secrets.ES_INDEX_TEST }} "--es_regex" "${{ github.event.inputs.regex }}" + + push_to_main_run: + if: ${{ github.event_name == 'push' }} + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.x' + + - name: Install Dependencies + run: | + python -m pip install --upgrade pip + pip install -r scripts/requirements/check_index.txt + + - name: Check Indexed Links - Live + run: | + python -u scripts/index_to_me.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ME_ES_HOST }} ${{ secrets.ME_ES_USER }} ${{ secrets.ME_ES_PASSWORD }} ${{ secrets.ES_INDEX_LIVE }} "--es_regex" "arm_gcc_clang|arm_mikroc|clocks|database|dspic|^images$|mikroe_utils|pic|preinit|riscv|schemas|unit_test_lib" + + - name: Check Indexed Links - Test + run: | + python -u scripts/index_to_me.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ME_ES_HOST }} ${{ secrets.ME_ES_USER }} ${{ secrets.ME_ES_PASSWORD }} ${{ secrets.ES_INDEX_TEST }} "--es_regex" "arm_gcc_clang|arm_mikroc|clocks|database|dspic|^images$|mikroe_utils|pic|preinit|riscv|schemas|unit_test_lib" + + scheduled_run: + if: ${{ github.event_name == 'schedule' }} + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.x' + + - name: Install Dependencies + run: | + python -m pip install --upgrade pip + pip install -r scripts/requirements/check_index.txt + + - name: Check Indexed Links - Live + run: | + python -u scripts/index_to_me.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ME_ES_HOST }} ${{ secrets.ME_ES_USER }} ${{ secrets.ME_ES_PASSWORD }} ${{ secrets.ES_INDEX_LIVE }} "--es_regex" "arm_gcc_clang|arm_mikroc|clocks|database|dspic|^images$|mikroe_utils|pic|preinit|riscv|schemas|unit_test_lib" + + - name: Check Indexed Links - Test + run: | + python -u scripts/index_to_me.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ME_ES_HOST }} ${{ secrets.ME_ES_USER }} ${{ secrets.ME_ES_PASSWORD }} ${{ secrets.ES_INDEX_TEST }} "--es_regex" "arm_gcc_clang|arm_mikroc|clocks|database|dspic|^images$|mikroe_utils|pic|preinit|riscv|schemas|unit_test_lib" diff --git a/scripts/classes/class_es.py b/scripts/classes/class_es.py index 149e4a1d6..731edae10 100644 --- a/scripts/classes/class_es.py +++ b/scripts/classes/class_es.py @@ -119,12 +119,21 @@ def create(self, doc_type, doc_id, doc_body): def update(self, doc_type, doc_id, doc_body): response = self.api_index(self.es_instance, self.index, doc_type, doc_id, doc_body) - if response['created'] and 'created' == response['result']: - print("%sWARNING: Asset \"%s\" created instead of updating. - %s" % (self.Colors.WARNING, doc_body['name'], doc_body['download_link'])) - elif not 'updated' == response['result']: - raise ValueError("%s%s failed to update on %s!" % (self.Colors.FAIL, doc_id, self.index)) + if doc_type: + if response['created'] and 'created' == response['result']: + print("%sWARNING: Asset \"%s\" created instead of updating. - %s" % (self.Colors.WARNING, doc_body['name'], doc_body['download_link'])) + elif not 'updated' == response['result']: + raise ValueError("%s%s failed to update on %s!" % (self.Colors.FAIL, doc_id, self.index)) + else: + print("%sINFO: Asset \"%s\" updated. - %s" % (self.Colors.OKGREEN, doc_body['name'], doc_body['download_link'])) + # For new DBP elasticsearch there is no 'created' field in response, so we just check 'result' else: - print("%sINFO: Asset \"%s\" updated. - %s" % (self.Colors.OKGREEN, doc_body['name'], doc_body['download_link'])) + if 'created' == response['result']: + print("%sWARNING: Asset \"%s\" created instead of updating. - %s" % (self.Colors.WARNING, doc_body['name'], doc_body['download_link'])) + elif not 'updated' == response['result']: + raise ValueError("%s%s failed to update on %s!" % (self.Colors.FAIL, doc_id, self.index)) + else: + print("%sINFO: Asset \"%s\" updated. - %s" % (self.Colors.OKGREEN, doc_body['name'], doc_body['download_link'])) def delete(self, doc_type, doc_id): response = self.es_instance.delete( diff --git a/scripts/index_to_me.py b/scripts/index_to_me.py index fea320b23..47df5a761 100644 --- a/scripts/index_to_me.py +++ b/scripts/index_to_me.py @@ -1,5 +1,4 @@ import sys, json, argparse, requests -# from elasticsearch import Elasticsearch import classes.class_gh as gh import classes.class_es as es @@ -8,17 +7,6 @@ type_skip = ['microchip_dfp', 'microchip_tp'] if __name__ == "__main__": - # First, check for arguments passed - def str2bool(v): - if isinstance(v, bool): - return v - if v.lower() in ('yes', 'true', 't', 'y', '1'): - return True - elif v.lower() in ('no', 'false', 'f', 'n', '0'): - return False - else: - raise argparse.ArgumentTypeError('Boolean value expected.') - # Get arguments parser = argparse.ArgumentParser(description="Upload directories as release assets.") parser.add_argument("gh_repo", help="Github repository name, e.g., 'username/repo'", type=str) @@ -26,22 +14,16 @@ def str2bool(v): parser.add_argument("es_host", help="ES instance host value", type=str) parser.add_argument("es_user", help="ES instance user value", type=str) parser.add_argument("es_password", help="ES instance password value", type=str) - # TODO https://kibana.mikroe.com parser.add_argument("me_es_host", help="MikroE ES instance host value", type=str) - # TODO sw-github parser.add_argument("me_es_user", help="MikroE ES instance user value", type=str) - # TODO sw-github334455 parser.add_argument("me_es_password", help="MikroE ES instance password value", type=str) - parser.add_argument("es_index", help="ES instance index value", type=str) # github_[test|live]_index parser.add_argument("--es_regex", help="Regex to use to fetch indexed items", type=str, default=".+") - # TODO parser.add_argument("--log_only", help="If True, will not fix broken links, just log them to std out", type=str2bool, default=False) - # TODO parser.add_argument("--index_package_names", help="If True, will add \"gh_package_name\" to indexed item", type=str2bool, default=True) args = parser.parse_args() - # es_instance = es.index( - # es_host=args.es_host, es_user=args.es_user, es_password=args.es_password, - # index=args.es_index, token=args.gh_token - # ) + es_instance = es.index( + es_host=args.es_host, es_user=args.es_user, es_password=args.es_password, + index=args.es_index, token=args.gh_token + ) me_es_instance = es.index( es_host=args.me_es_host, es_user=args.me_es_user, es_password=args.me_es_password, @@ -56,13 +38,11 @@ def str2bool(v): 'Authorization': f'token {args.gh_token}' } - err = False for indexed_item in es_instance.indexed_items: if indexed_item['source']['type'] in type_skip: continue asset_status = requests.get(indexed_item['source']['download_link'], headers=headers) if es_instance.Status.ERROR.value == asset_status.status_code: ## code 404 - error, reindex with correct download link - err = True print("%sERROR: Asset \"%s\" download link is incorrect. - %s" % (es_instance.Colors.FAIL, indexed_item['source']['name'], indexed_item['source']['download_link'])) if 'gh_package_name' in indexed_item['source']: url = gh_instance.asset_fetch_url_api(indexed_item['source']['gh_package_name'], loose=False) @@ -83,8 +63,6 @@ def str2bool(v): print("%sINFO: Updated \"gh_package_name\" for %s" % (es_instance.Colors.UNDERLINE, indexed_item['source']['name'])) print("%sOK: Asset \"%s\" download link is correct. - %s" % (es_instance.Colors.OKBLUE, indexed_item['source']['name'], indexed_item['source']['download_link'])) - # TODO - index to me_es - me_es_instance.update(indexed_item['doc']['type'], indexed_item['doc']['id'], indexed_item['source']) - - if err and args.log_only: - sys.exit(-1) \ No newline at end of file + # For new elasticsearch DBP it is crucial not to use doc_type for indexing + me_es_instance.update(None, indexed_item['doc']['id'], indexed_item['source']) + print("%sINFO: Copied \"%s\" index from AWS to DBS" % (es_instance.Colors.UNDERLINE, indexed_item['source']['name'], )) From ecfbb2f6c814abb053c58cb83e164d037154cb1a Mon Sep 17 00:00:00 2001 From: "ivan.ruzavin" Date: Thu, 14 Nov 2024 12:56:05 +0100 Subject: [PATCH 3/5] Cjanged --- scripts/index_to_me.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/index_to_me.py b/scripts/index_to_me.py index 47df5a761..8ee6443ad 100644 --- a/scripts/index_to_me.py +++ b/scripts/index_to_me.py @@ -65,4 +65,4 @@ # For new elasticsearch DBP it is crucial not to use doc_type for indexing me_es_instance.update(None, indexed_item['doc']['id'], indexed_item['source']) - print("%sINFO: Copied \"%s\" index from AWS to DBS" % (es_instance.Colors.UNDERLINE, indexed_item['source']['name'], )) + print("%sINFO: Copied \"%s\" index from AWS to DBS" % (me_es_instance.Colors.UNDERLINE, indexed_item['source']['name'], )) From 48159d07a9b7737e0de4d5ddc437352b53bc5ac4 Mon Sep 17 00:00:00 2001 From: "ivan.ruzavin" Date: Fri, 15 Nov 2024 08:56:27 +0100 Subject: [PATCH 4/5] Added index changes to the original workflow --- .github/workflows/checkIndexes.yaml | 15 ++-- .github/workflows/copyIndexesToDBP.yaml | 94 ------------------------- scripts/check_indexes.py | 12 ++++ scripts/index_to_me.py | 68 ------------------ 4 files changed, 21 insertions(+), 168 deletions(-) delete mode 100644 .github/workflows/copyIndexesToDBP.yaml delete mode 100644 scripts/index_to_me.py diff --git a/.github/workflows/checkIndexes.yaml b/.github/workflows/checkIndexes.yaml index 53c505f2d..d634a5438 100644 --- a/.github/workflows/checkIndexes.yaml +++ b/.github/workflows/checkIndexes.yaml @@ -26,6 +26,9 @@ on: schedule: - cron: "0/30 7-16 * * 1-5" # Every 30 minutes, between 07:00 AM and 04:59 PM, Monday through Friday +env: + GLOBAL_REGEX: "arm_gcc_clang|arm_mikroc|clocks|database|dspic|^images$|mikroe_utils|pic|preinit|riscv|schemas|unit_test_lib" + jobs: manual_run: if: ${{ github.event_name == 'workflow_dispatch' }} @@ -50,7 +53,7 @@ jobs: FIX_ACTION=${{ github.event.inputs.fix }} # Capture the fix input LOG_ONLY=$([[ "$FIX_ACTION" == "false" ]] && echo true || echo false) # Negate the fix input echo "LOG_ONLY is set to $LOG_ONLY" - python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ES_INDEX_LIVE }} "--es_regex" "${{ github.event.inputs.regex }}" "--log_only" "$LOG_ONLY" + python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ME_ES_HOST }} ${{ secrets.ME_ES_USER }} ${{ secrets.ME_ES_PASSWORD }} ${{ secrets.ES_INDEX_LIVE }} "--es_regex" "${{ github.event.inputs.regex }}" "--log_only" "$LOG_ONLY" continue-on-error: true # Ensure the workflow continues - name: Check Indexed Links - Test @@ -59,7 +62,7 @@ jobs: FIX_ACTION=${{ github.event.inputs.fix }} # Capture the fix input LOG_ONLY=$([[ "$FIX_ACTION" == "false" ]] && echo true || echo false) # Negate the fix input echo "LOG_ONLY is set to $LOG_ONLY" - python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ES_INDEX_TEST }} "--es_regex" "${{ github.event.inputs.regex }}" "--log_only" "$LOG_ONLY" + python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ME_ES_HOST }} ${{ secrets.ME_ES_USER }} ${{ secrets.ME_ES_PASSWORD }} ${{ secrets.ES_INDEX_TEST }} "--es_regex" "${{ github.event.inputs.regex }}" "--log_only" "$LOG_ONLY" continue-on-error: true # Ensure the workflow continues push_to_main_run: @@ -81,12 +84,12 @@ jobs: - name: Check Indexed Links - Live run: | - python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ES_INDEX_LIVE }} "--es_regex" "arm_gcc_clang|arm_mikroc|clocks|database|dspic|^images$|mikroe_utils|pic|preinit|riscv|schemas|unit_test_lib" + python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ME_ES_HOST }} ${{ secrets.ME_ES_USER }} ${{ secrets.ME_ES_PASSWORD }} ${{ secrets.ES_INDEX_LIVE }} "--es_regex" "$GLOBAL_REGEX" continue-on-error: true # Ensure the workflow continues - name: Check Indexed Links - Test run: | - python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ES_INDEX_TEST }} "--es_regex" "arm_gcc_clang|arm_mikroc|clocks|database|dspic|^images$|mikroe_utils|pic|preinit|riscv|schemas|unit_test_lib" + python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ME_ES_HOST }} ${{ secrets.ME_ES_USER }} ${{ secrets.ME_ES_PASSWORD }} ${{ secrets.ES_INDEX_TEST }} "--es_regex" "$GLOBAL_REGEX" continue-on-error: true # Ensure the workflow continues scheduled_run: @@ -108,10 +111,10 @@ jobs: - name: Check Indexed Links - Live run: | - python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ES_INDEX_LIVE }} "--es_regex" "arm_gcc_clang|arm_mikroc|clocks|database|dspic|^images$|mikroe_utils|pic|preinit|riscv|schemas|unit_test_lib" + python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ME_ES_HOST }} ${{ secrets.ME_ES_USER }} ${{ secrets.ME_ES_PASSWORD }} ${{ secrets.ES_INDEX_LIVE }} "--es_regex" "$GLOBAL_REGEX" continue-on-error: true # Ensure the workflow continues - name: Check Indexed Links - Test run: | - python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ES_INDEX_TEST }} "--es_regex" "arm_gcc_clang|arm_mikroc|clocks|database|dspic|^images$|mikroe_utils|pic|preinit|riscv|schemas|unit_test_lib" + python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ME_ES_HOST }} ${{ secrets.ME_ES_USER }} ${{ secrets.ME_ES_PASSWORD }} ${{ secrets.ES_INDEX_TEST }} "--es_regex" "$GLOBAL_REGEX" continue-on-error: true # Ensure the workflow continues diff --git a/.github/workflows/copyIndexesToDBP.yaml b/.github/workflows/copyIndexesToDBP.yaml deleted file mode 100644 index 6ab0c410b..000000000 --- a/.github/workflows/copyIndexesToDBP.yaml +++ /dev/null @@ -1,94 +0,0 @@ -name: Copy Indexes from AWS to DBP - -on: - workflow_dispatch: - inputs: - select_index: - type: choice - description: Copy Test and/or Live ES indexed items - options: - - Test - - Live - - Both - regex: - type: string - description: Regex to use when searching for indexed items - default: "arm_gcc_clang|arm_mikroc|clocks|database|dspic|^images$|mikroe_utils|pic|preinit|riscv|schemas|unit_test_lib" - -jobs: - manual_run: - if: ${{ github.event_name == 'workflow_dispatch' }} - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.x' - - - name: Install Dependencies - run: | - python -m pip install --upgrade pip - pip install -r scripts/requirements/check_index.txt - - - name: Copy Indexed Links - Live - if: ${{ github.event.inputs.select_index == 'Live' || github.event.inputs.select_index == 'Both' }} - run: | - python -u scripts/index_to_me.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ME_ES_HOST }} ${{ secrets.ME_ES_USER }} ${{ secrets.ME_ES_PASSWORD }} ${{ secrets.ES_INDEX_LIVE }} "--es_regex" "${{ github.event.inputs.regex }}" - - - name: Copy Indexed Links - Test - if: ${{ github.event.inputs.select_index == 'Test' || github.event.inputs.select_index == 'Both' }} - run: | - python -u scripts/index_to_me.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ME_ES_HOST }} ${{ secrets.ME_ES_USER }} ${{ secrets.ME_ES_PASSWORD }} ${{ secrets.ES_INDEX_TEST }} "--es_regex" "${{ github.event.inputs.regex }}" - - push_to_main_run: - if: ${{ github.event_name == 'push' }} - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.x' - - - name: Install Dependencies - run: | - python -m pip install --upgrade pip - pip install -r scripts/requirements/check_index.txt - - - name: Check Indexed Links - Live - run: | - python -u scripts/index_to_me.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ME_ES_HOST }} ${{ secrets.ME_ES_USER }} ${{ secrets.ME_ES_PASSWORD }} ${{ secrets.ES_INDEX_LIVE }} "--es_regex" "arm_gcc_clang|arm_mikroc|clocks|database|dspic|^images$|mikroe_utils|pic|preinit|riscv|schemas|unit_test_lib" - - - name: Check Indexed Links - Test - run: | - python -u scripts/index_to_me.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ME_ES_HOST }} ${{ secrets.ME_ES_USER }} ${{ secrets.ME_ES_PASSWORD }} ${{ secrets.ES_INDEX_TEST }} "--es_regex" "arm_gcc_clang|arm_mikroc|clocks|database|dspic|^images$|mikroe_utils|pic|preinit|riscv|schemas|unit_test_lib" - - scheduled_run: - if: ${{ github.event_name == 'schedule' }} - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.x' - - - name: Install Dependencies - run: | - python -m pip install --upgrade pip - pip install -r scripts/requirements/check_index.txt - - - name: Check Indexed Links - Live - run: | - python -u scripts/index_to_me.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ME_ES_HOST }} ${{ secrets.ME_ES_USER }} ${{ secrets.ME_ES_PASSWORD }} ${{ secrets.ES_INDEX_LIVE }} "--es_regex" "arm_gcc_clang|arm_mikroc|clocks|database|dspic|^images$|mikroe_utils|pic|preinit|riscv|schemas|unit_test_lib" - - - name: Check Indexed Links - Test - run: | - python -u scripts/index_to_me.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ME_ES_HOST }} ${{ secrets.ME_ES_USER }} ${{ secrets.ME_ES_PASSWORD }} ${{ secrets.ES_INDEX_TEST }} "--es_regex" "arm_gcc_clang|arm_mikroc|clocks|database|dspic|^images$|mikroe_utils|pic|preinit|riscv|schemas|unit_test_lib" diff --git a/scripts/check_indexes.py b/scripts/check_indexes.py index ad1fbf1d8..6805d67d9 100644 --- a/scripts/check_indexes.py +++ b/scripts/check_indexes.py @@ -25,6 +25,9 @@ def str2bool(v): parser.add_argument("es_host", help="ES instance host value", type=str) parser.add_argument("es_user", help="ES instance user value", type=str) parser.add_argument("es_password", help="ES instance password value", type=str) + parser.add_argument("me_es_host", help="MikroE ES instance host value", type=str) + parser.add_argument("me_es_user", help="MikroE ES instance user value", type=str) + parser.add_argument("me_es_password", help="MikroE ES instance password value", type=str) parser.add_argument("es_index", help="ES instance index value", type=str) parser.add_argument("--es_regex", help="Regex to use to fetch indexed items", type=str, default=".+") parser.add_argument("--log_only", help="If True, will not fix broken links, just log them to std out", type=str2bool, default=False) @@ -36,6 +39,11 @@ def str2bool(v): index=args.es_index, token=args.gh_token ) + me_es_instance = es.index( + es_host=args.me_es_host, es_user=args.me_es_user, es_password=args.me_es_password, + index=args.es_index, token=args.gh_token + ) + gh_instance = gh.repo(args.gh_repo, args.gh_token) es_instance.fetch(regex=args.es_regex) @@ -73,5 +81,9 @@ def str2bool(v): print("%sINFO: Updated \"gh_package_name\" for %s" % (es_instance.Colors.UNDERLINE, indexed_item['source']['name'])) print("%sOK: Asset \"%s\" download link is correct. - %s" % (es_instance.Colors.OKBLUE, indexed_item['source']['name'], indexed_item['source']['download_link'])) + # For new elasticsearch DBP it is crucial not to use doc_type for indexing + me_es_instance.update(None, indexed_item['doc']['id'], indexed_item['source']) + print("%sINFO: Copied \"%s\" index from AWS to DBS" % (me_es_instance.Colors.UNDERLINE, indexed_item['source']['name'], )) + if err and args.log_only: sys.exit(-1) diff --git a/scripts/index_to_me.py b/scripts/index_to_me.py deleted file mode 100644 index 8ee6443ad..000000000 --- a/scripts/index_to_me.py +++ /dev/null @@ -1,68 +0,0 @@ -import sys, json, argparse, requests - -import classes.class_gh as gh -import classes.class_es as es - -# Skip packages with these types -type_skip = ['microchip_dfp', 'microchip_tp'] - -if __name__ == "__main__": - # Get arguments - parser = argparse.ArgumentParser(description="Upload directories as release assets.") - parser.add_argument("gh_repo", help="Github repository name, e.g., 'username/repo'", type=str) - parser.add_argument("gh_token", help="GitHub Token", type=str) - parser.add_argument("es_host", help="ES instance host value", type=str) - parser.add_argument("es_user", help="ES instance user value", type=str) - parser.add_argument("es_password", help="ES instance password value", type=str) - parser.add_argument("me_es_host", help="MikroE ES instance host value", type=str) - parser.add_argument("me_es_user", help="MikroE ES instance user value", type=str) - parser.add_argument("me_es_password", help="MikroE ES instance password value", type=str) - parser.add_argument("--es_regex", help="Regex to use to fetch indexed items", type=str, default=".+") - args = parser.parse_args() - - es_instance = es.index( - es_host=args.es_host, es_user=args.es_user, es_password=args.es_password, - index=args.es_index, token=args.gh_token - ) - - me_es_instance = es.index( - es_host=args.me_es_host, es_user=args.me_es_user, es_password=args.me_es_password, - index=args.es_index, token=args.gh_token - ) - - gh_instance = gh.repo(args.gh_repo, args.gh_token) - - es_instance.fetch(regex=args.es_regex) - - headers = { - 'Authorization': f'token {args.gh_token}' - } - - for indexed_item in es_instance.indexed_items: - if indexed_item['source']['type'] in type_skip: - continue - asset_status = requests.get(indexed_item['source']['download_link'], headers=headers) - if es_instance.Status.ERROR.value == asset_status.status_code: ## code 404 - error, reindex with correct download link - print("%sERROR: Asset \"%s\" download link is incorrect. - %s" % (es_instance.Colors.FAIL, indexed_item['source']['name'], indexed_item['source']['download_link'])) - if 'gh_package_name' in indexed_item['source']: - url = gh_instance.asset_fetch_url_api(indexed_item['source']['gh_package_name'], loose=False) - indexed_item['source']['download_link'] = url - es_instance.update(indexed_item['doc']['type'], indexed_item['doc']['id'], indexed_item['source']) - else: - print("%sWARNING: Asset \"%s\" has no \"gh_package_name\" in the index." % (es_instance.Colors.WARNING, indexed_item['source']['name'])) - else: ## code 200 - success, no need to reindex - package_name = (json.loads(asset_status.text))['name'] - if 'gh_package_name' not in indexed_item['source']: - indexed_item['source'].update({"gh_package_name": package_name}) - es_instance.update(indexed_item['doc']['type'], indexed_item['doc']['id'], indexed_item['source']) - print("%sINFO: Added \"gh_package_name\" to %s" % (es_instance.Colors.UNDERLINE, indexed_item['source']['name'])) - else: - if package_name != indexed_item['source']['gh_package_name']: - indexed_item['source']['gh_package_name'] = package_name - es_instance.update(indexed_item['doc']['type'], indexed_item['doc']['id'], indexed_item['source']) - print("%sINFO: Updated \"gh_package_name\" for %s" % (es_instance.Colors.UNDERLINE, indexed_item['source']['name'])) - print("%sOK: Asset \"%s\" download link is correct. - %s" % (es_instance.Colors.OKBLUE, indexed_item['source']['name'], indexed_item['source']['download_link'])) - - # For new elasticsearch DBP it is crucial not to use doc_type for indexing - me_es_instance.update(None, indexed_item['doc']['id'], indexed_item['source']) - print("%sINFO: Copied \"%s\" index from AWS to DBS" % (me_es_instance.Colors.UNDERLINE, indexed_item['source']['name'], )) From 0bffe6e915d024f6f174ce1d06e3caba91c0ab47 Mon Sep 17 00:00:00 2001 From: "ivan.ruzavin" Date: Fri, 15 Nov 2024 09:06:00 +0100 Subject: [PATCH 5/5] Updated comment --- scripts/check_indexes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/check_indexes.py b/scripts/check_indexes.py index 6805d67d9..b4ff791ef 100644 --- a/scripts/check_indexes.py +++ b/scripts/check_indexes.py @@ -83,7 +83,7 @@ def str2bool(v): # For new elasticsearch DBP it is crucial not to use doc_type for indexing me_es_instance.update(None, indexed_item['doc']['id'], indexed_item['source']) - print("%sINFO: Copied \"%s\" index from AWS to DBS" % (me_es_instance.Colors.UNDERLINE, indexed_item['source']['name'], )) + print("%sINFO: Copied \"%s\" index from AWS to DBS" % (me_es_instance.Colors.UNDERLINE, indexed_item['source']['name'])) if err and args.log_only: sys.exit(-1)