Merge pull request #255 from MikroElektronika/update/set-new-url-links

Updated URL assigning approach

scripts/check_indexes.py

@@ -6,6 +6,17 @@
 # Legacy packages for NECTO version 7.0.4 and lower
 legacy_packages = ["clocks", "schemas", "database", "images", "images_sdk"]
 
+def assign_urls(indexed_item, gh_instance, es_instance):
+    # Check if download link for all github assets is direct github download url
+    url = gh_instance.asset_fetch_url_browser(indexed_item['source']['gh_package_name'], loose=False)
+    url_api = gh_instance.asset_fetch_url_api(indexed_item['source']['gh_package_name'], loose=False)
+    # For non-github packages url is None, so we should skip it
+    if indexed_item['source']['download_link'] != url and url != None:
+        indexed_item['source']['download_link'] = url
+        indexed_item['source']['download_link_api'] = url_api
+        es_instance.update(None, indexed_item['doc']['id'], indexed_item['source'])
+        print("%sINFO: Updated \"download_link\" for %s" % (es_instance.Colors.UNDERLINE, indexed_item['source']['name']))
+
 if __name__ == "__main__":
     # First, check for arguments passed
     def str2bool(v):
@@ -56,23 +67,22 @@ def str2bool(v):
 
     err = False
     for indexed_item in es_instance.indexed_items:
-        asset_status = requests.get(indexed_item['source']['download_link'], headers=headers)
+        if 'download_link_api' in indexed_item['source']:
+            asset_status = requests.get(indexed_item['source']['download_link_api'], headers=headers)
+        else:
+            asset_status = requests.get(indexed_item['source']['download_link'], headers=headers)
         if es_instance.Status.ERROR.value == asset_status.status_code: ## code 404 - error, reindex with correct download link
             err = True
             print("%sERROR: Asset \"%s\" download link is incorrect. - %s" % (es_instance.Colors.FAIL, indexed_item['source']['name'], indexed_item['source']['download_link']))
             if not args.log_only:
                 if 'gh_package_name' in indexed_item['source']:
-                    url = gh_instance.asset_fetch_url_api(indexed_item['source']['gh_package_name'], loose=False)
-                    indexed_item['source']['download_link'] = url
-                    es_instance.update(None, indexed_item['doc']['id'], indexed_item['source'])
+                    # Assign correct URLs
+                    assign_urls(indexed_item, gh_instance, es_instance)
                 else:
                     print("%sWARNING: Asset \"%s\" has no \"gh_package_name\" in the index." % (es_instance.Colors.WARNING, indexed_item['source']['name']))
         else: ## code 200 - success, no need to reindex
             if args.index_package_names:
-                if 'download.microchip.com' in indexed_item['source']['download_link'] or 'amazonaws' in indexed_item['source']['download_link']:
-                    package_name = indexed_item['source']['name']
-                else:
-                    package_name = (json.loads(asset_status.text))['name']
+                package_name = f'{indexed_item['source']['name']}.7z'
                 if 'gh_package_name' not in indexed_item['source']:
                     indexed_item['source'].update({"gh_package_name": package_name})
                     es_instance.update(None, indexed_item['doc']['id'], indexed_item['source'])
@@ -82,6 +92,9 @@ def str2bool(v):
                         indexed_item['source']['gh_package_name'] = package_name
                         es_instance.update(None, indexed_item['doc']['id'], indexed_item['source'])
                         print("%sINFO: Updated \"gh_package_name\" for %s" % (es_instance.Colors.UNDERLINE, indexed_item['source']['name']))
+
+                # Assign correct URLs
+                assign_urls(indexed_item, gh_instance, es_instance)
             print("%sOK: Asset \"%s\" download link is correct. - %s" % (es_instance.Colors.OKBLUE, indexed_item['source']['name'], indexed_item['source']['download_link']))
 
         if 'ES_HOST_LEGACY' in os.environ and 'ES_USER_LEGACY' in os.environ and 'ES_PASSWORD_LEGACY':

scripts/index.py

@@ -284,7 +284,8 @@ def index_release_to_elasticsearch(es : Elasticsearch, index_name, release_detai
                 'created_at': asset['created_at'],
                 'updated_at': asset['updated_at'],
                 'category': "MCU Package",
-                'download_link': asset['url'],
+                'download_link': asset['browser_download_url'],
+                'download_link_api': asset['url'],
                 'package_changed' : check_version,
                 'install_location' : "%APPLICATION_DATA_DIR%/clocks.json",
                 'gh_package_name': "clocks.json"
@@ -305,7 +306,8 @@ def index_release_to_elasticsearch(es : Elasticsearch, index_name, release_detai
                 'created_at': asset['created_at'],
                 'updated_at': asset['updated_at'],
                 'category': "MCU Package",
-                'download_link': asset['url'],
+                'download_link': asset['browser_download_url'],
+                'download_link_api': asset['url'],
                 'package_changed' : check_version,
                 'install_location' : f"%APPLICATION_DATA_DIR%/schemas{suffix}.json",
                 'gh_package_name': f"schemas{suffix}.json"
@@ -358,7 +360,8 @@ def index_release_to_elasticsearch(es : Elasticsearch, index_name, release_detai
                     'published_at': published_at,
                     'hash': current_hash,
                     'category': metadata_item['category'],
-                    'download_link': asset['url'],
+                    'download_link': asset['browser_download_url'],
+                    'download_link_api': asset['url'],
                     'package_changed' : update_package or force,
                     'install_location' : metadata_item['install_location'],
                     'gh_package_name': os.path.splitext(os.path.basename(asset['name']))[0]