diff --git a/.github/workflows/checkIndexes.yaml b/.github/workflows/checkIndexes.yaml index 53c505f2d..d634a5438 100644 --- a/.github/workflows/checkIndexes.yaml +++ b/.github/workflows/checkIndexes.yaml @@ -26,6 +26,9 @@ on: schedule: - cron: "0/30 7-16 * * 1-5" # Every 30 minutes, between 07:00 AM and 04:59 PM, Monday through Friday +env: + GLOBAL_REGEX: "arm_gcc_clang|arm_mikroc|clocks|database|dspic|^images$|mikroe_utils|pic|preinit|riscv|schemas|unit_test_lib" + jobs: manual_run: if: ${{ github.event_name == 'workflow_dispatch' }} @@ -50,7 +53,7 @@ jobs: FIX_ACTION=${{ github.event.inputs.fix }} # Capture the fix input LOG_ONLY=$([[ "$FIX_ACTION" == "false" ]] && echo true || echo false) # Negate the fix input echo "LOG_ONLY is set to $LOG_ONLY" - python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ES_INDEX_LIVE }} "--es_regex" "${{ github.event.inputs.regex }}" "--log_only" "$LOG_ONLY" + python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ME_ES_HOST }} ${{ secrets.ME_ES_USER }} ${{ secrets.ME_ES_PASSWORD }} ${{ secrets.ES_INDEX_LIVE }} "--es_regex" "${{ github.event.inputs.regex }}" "--log_only" "$LOG_ONLY" continue-on-error: true # Ensure the workflow continues - name: Check Indexed Links - Test @@ -59,7 +62,7 @@ jobs: FIX_ACTION=${{ github.event.inputs.fix }} # Capture the fix input LOG_ONLY=$([[ "$FIX_ACTION" == "false" ]] && echo true || echo false) # Negate the fix input echo "LOG_ONLY is set to $LOG_ONLY" - python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ES_INDEX_TEST }} "--es_regex" "${{ github.event.inputs.regex }}" "--log_only" "$LOG_ONLY" + python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ME_ES_HOST }} ${{ secrets.ME_ES_USER }} ${{ secrets.ME_ES_PASSWORD }} ${{ secrets.ES_INDEX_TEST }} "--es_regex" "${{ github.event.inputs.regex }}" "--log_only" "$LOG_ONLY" continue-on-error: true # Ensure the workflow continues push_to_main_run: @@ -81,12 +84,12 @@ jobs: - name: Check Indexed Links - Live run: | - python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ES_INDEX_LIVE }} "--es_regex" "arm_gcc_clang|arm_mikroc|clocks|database|dspic|^images$|mikroe_utils|pic|preinit|riscv|schemas|unit_test_lib" + python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ME_ES_HOST }} ${{ secrets.ME_ES_USER }} ${{ secrets.ME_ES_PASSWORD }} ${{ secrets.ES_INDEX_LIVE }} "--es_regex" "$GLOBAL_REGEX" continue-on-error: true # Ensure the workflow continues - name: Check Indexed Links - Test run: | - python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ES_INDEX_TEST }} "--es_regex" "arm_gcc_clang|arm_mikroc|clocks|database|dspic|^images$|mikroe_utils|pic|preinit|riscv|schemas|unit_test_lib" + python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ME_ES_HOST }} ${{ secrets.ME_ES_USER }} ${{ secrets.ME_ES_PASSWORD }} ${{ secrets.ES_INDEX_TEST }} "--es_regex" "$GLOBAL_REGEX" continue-on-error: true # Ensure the workflow continues scheduled_run: @@ -108,10 +111,10 @@ jobs: - name: Check Indexed Links - Live run: | - python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ES_INDEX_LIVE }} "--es_regex" "arm_gcc_clang|arm_mikroc|clocks|database|dspic|^images$|mikroe_utils|pic|preinit|riscv|schemas|unit_test_lib" + python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ME_ES_HOST }} ${{ secrets.ME_ES_USER }} ${{ secrets.ME_ES_PASSWORD }} ${{ secrets.ES_INDEX_LIVE }} "--es_regex" "$GLOBAL_REGEX" continue-on-error: true # Ensure the workflow continues - name: Check Indexed Links - Test run: | - python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ES_INDEX_TEST }} "--es_regex" "arm_gcc_clang|arm_mikroc|clocks|database|dspic|^images$|mikroe_utils|pic|preinit|riscv|schemas|unit_test_lib" + python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ME_ES_HOST }} ${{ secrets.ME_ES_USER }} ${{ secrets.ME_ES_PASSWORD }} ${{ secrets.ES_INDEX_TEST }} "--es_regex" "$GLOBAL_REGEX" continue-on-error: true # Ensure the workflow continues diff --git a/scripts/check_indexes.py b/scripts/check_indexes.py index ad1fbf1d8..b4ff791ef 100644 --- a/scripts/check_indexes.py +++ b/scripts/check_indexes.py @@ -25,6 +25,9 @@ def str2bool(v): parser.add_argument("es_host", help="ES instance host value", type=str) parser.add_argument("es_user", help="ES instance user value", type=str) parser.add_argument("es_password", help="ES instance password value", type=str) + parser.add_argument("me_es_host", help="MikroE ES instance host value", type=str) + parser.add_argument("me_es_user", help="MikroE ES instance user value", type=str) + parser.add_argument("me_es_password", help="MikroE ES instance password value", type=str) parser.add_argument("es_index", help="ES instance index value", type=str) parser.add_argument("--es_regex", help="Regex to use to fetch indexed items", type=str, default=".+") parser.add_argument("--log_only", help="If True, will not fix broken links, just log them to std out", type=str2bool, default=False) @@ -36,6 +39,11 @@ def str2bool(v): index=args.es_index, token=args.gh_token ) + me_es_instance = es.index( + es_host=args.me_es_host, es_user=args.me_es_user, es_password=args.me_es_password, + index=args.es_index, token=args.gh_token + ) + gh_instance = gh.repo(args.gh_repo, args.gh_token) es_instance.fetch(regex=args.es_regex) @@ -73,5 +81,9 @@ def str2bool(v): print("%sINFO: Updated \"gh_package_name\" for %s" % (es_instance.Colors.UNDERLINE, indexed_item['source']['name'])) print("%sOK: Asset \"%s\" download link is correct. - %s" % (es_instance.Colors.OKBLUE, indexed_item['source']['name'], indexed_item['source']['download_link'])) + # For new elasticsearch DBP it is crucial not to use doc_type for indexing + me_es_instance.update(None, indexed_item['doc']['id'], indexed_item['source']) + print("%sINFO: Copied \"%s\" index from AWS to DBS" % (me_es_instance.Colors.UNDERLINE, indexed_item['source']['name'])) + if err and args.log_only: sys.exit(-1) diff --git a/scripts/classes/class_es.py b/scripts/classes/class_es.py index 149e4a1d6..731edae10 100644 --- a/scripts/classes/class_es.py +++ b/scripts/classes/class_es.py @@ -119,12 +119,21 @@ def create(self, doc_type, doc_id, doc_body): def update(self, doc_type, doc_id, doc_body): response = self.api_index(self.es_instance, self.index, doc_type, doc_id, doc_body) - if response['created'] and 'created' == response['result']: - print("%sWARNING: Asset \"%s\" created instead of updating. - %s" % (self.Colors.WARNING, doc_body['name'], doc_body['download_link'])) - elif not 'updated' == response['result']: - raise ValueError("%s%s failed to update on %s!" % (self.Colors.FAIL, doc_id, self.index)) + if doc_type: + if response['created'] and 'created' == response['result']: + print("%sWARNING: Asset \"%s\" created instead of updating. - %s" % (self.Colors.WARNING, doc_body['name'], doc_body['download_link'])) + elif not 'updated' == response['result']: + raise ValueError("%s%s failed to update on %s!" % (self.Colors.FAIL, doc_id, self.index)) + else: + print("%sINFO: Asset \"%s\" updated. - %s" % (self.Colors.OKGREEN, doc_body['name'], doc_body['download_link'])) + # For new DBP elasticsearch there is no 'created' field in response, so we just check 'result' else: - print("%sINFO: Asset \"%s\" updated. - %s" % (self.Colors.OKGREEN, doc_body['name'], doc_body['download_link'])) + if 'created' == response['result']: + print("%sWARNING: Asset \"%s\" created instead of updating. - %s" % (self.Colors.WARNING, doc_body['name'], doc_body['download_link'])) + elif not 'updated' == response['result']: + raise ValueError("%s%s failed to update on %s!" % (self.Colors.FAIL, doc_id, self.index)) + else: + print("%sINFO: Asset \"%s\" updated. - %s" % (self.Colors.OKGREEN, doc_body['name'], doc_body['download_link'])) def delete(self, doc_type, doc_id): response = self.es_instance.delete(