Finalized update for indexing elastic search values

.github/workflows/index.yaml

@@ -0,0 +1,33 @@
+name: Index Latest Release
+
+on:
+  workflow_dispatch:
+
+jobs:
+  index:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.x'
+
+      - name: Install Dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install aiohttp
+          pip install aiofiles
+          pip install requests
+          pip install elasticsearch==7.13.4
+          sudo apt-get install p7zip-full
+
+      - name: Run Index Script
+        env:
+          ES_HOST: ${{ secrets.ES_HOST }}
+          ES_INDEX: ${{ secrets.ES_INDEX }}
+          ES_USER: ${{ secrets.ES_USER }}
+          ES_PASSWORD: ${{ secrets.ES_PASSWORD }}
+        run: python -u scripts/index.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }}

.github/workflows/release.yaml

@@ -1,5 +1,5 @@
-name: Release Asset Upload Workflow
-              
+name: Release Asset Upload
+
 on:
   release:
     types: [published]
@@ -19,12 +19,8 @@ jobs:
       - name: Install Dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install aiohttp aiofiles requests py7zr
-          sudo apt-get install p7zip-full
-          pip install elasticsearch
+          pip install requests
+          pip install py7zr
 
       - name: Run Release Script
-        env:
-          ES_HOST: ${{ secrets.ES_HOST }}
-          ES_INDEX: ${{ secrets.ES_INDEX }}
-        run: python -u scripts/package.py ${{ secrets.GITHUB_TOKEN }} ${{ github.repository }} ${{ github.event.release.tag_name }}
+        run: python -u scripts/package.py ${{ secrets.GITHUB_TOKEN }} ${{ github.repository }}

.gitignore

@@ -0,0 +1,6 @@
+*.asm
+.vscode
+output
+__pycache__
+*.7z
+tmp

scripts/index.py

@@ -0,0 +1,132 @@
+import os, time, argparse, requests
+from elasticsearch import Elasticsearch
+
+import support as support
+
+# Gets latest release headers from repository
+def get_headers(api, token):
+    if api:
+        return {
+            'Authorization': f'token {token}'
+        }
+    else:
+        return {
+            'Authorization': f'Bearer {token}',
+            'Accept': 'application/octet-stream'
+        }
+
+# Function to fetch release details from GitHub
+def fetch_release_details(repo, token):
+    api_headers = get_headers(True, token)
+    url = f'https://api.github.com/repos/{repo}/releases'
+    response = requests.get(url, headers=api_headers)
+    response.raise_for_status()  # Raise an exception for HTTP errors
+    return support.get_latest_release(response.json())
+
+# Function to fetch content as JSON from the link
+def fetch_json_data(download_link, token):
+    """
+    Fetches JSON data from the specified URL using an authorization token and returns it as a dictionary.
+
+    Args:
+        download_link (str): URL from which to fetch the JSON data.
+
+    Returns:
+        tuple: The first element is a dictionary containing the JSON data (or None in case of failure),
+                the second element is an error message or None if no errors occurred.
+    """
+    headers = get_headers(False, token)
+
+    try:
+        response = requests.get(download_link, headers=headers)
+        response.raise_for_status()
+        return response.json(), None
+    except requests.RequestException as e:
+        print(f"Error fetching JSON data: {e}")
+        return None, str(e)
+
+# Function to find an item by name
+def find_item_by_name(items, name):
+    for item in items:
+        if item['name'] == name:
+            return item
+    return None
+
+# Function to index release details into Elasticsearch
+def index_release_to_elasticsearch(es : Elasticsearch, index_name, release_details, token):
+    version = None
+    for asset in release_details.get('assets', []):
+        if 'mikrosdk.7z' == asset['name']:
+            # Download the current mikroSDK asset in order to read the version
+            support.extract_archive_from_url(
+                asset['url'],
+                os.path.join(os.path.dirname(__file__), 'tmp'),
+                token
+            )
+
+            # Then fetch version from manifest file
+            version = support.fetch_version_from_asset(os.path.join(os.path.dirname(__file__), 'tmp'))
+            break
+    for asset in release_details.get('assets', []):
+        doc = None
+        name_without_extension = os.path.splitext(os.path.basename(asset['name']))[0]
+        if 'mikrosdk' == name_without_extension:
+            doc = {
+                'name': name_without_extension,
+                'display_name': "mikroSDK",
+                'author': 'MIKROE',
+                'hidden': False,
+                'type': 'sdk',
+                'version': version,
+                'created_at' : asset['created_at'],
+                'updated_at' : asset['updated_at'],
+                'category': 'Software Development Kit',
+                'download_link': asset['url'],  # Adjust as needed for actual URL
+                'package_changed': True
+            }
+        elif 'templates' == name_without_extension:
+            doc = {
+                "name": name_without_extension,
+                "version" : version,
+                "display_name" : "NECTO project templates",
+                "hidden" : True,
+                "vendor" : "MIKROE",
+                "type" : "application",
+                "location" : f"necto/%os_type%/{asset['name']}.7z",
+                "install_location" : "%APPLICATION_DATA_DIR%",
+                "dependencies": []
+            }
+
+        # Index the document
+        if doc:
+            resp = es.index(index=index_name, doc_type='necto_package', id=name_without_extension, body=doc)
+            print(f"{resp["result"]} {resp['_id']}")
+
+if __name__ == '__main__':
+    # Get arguments
+    parser = argparse.ArgumentParser(description="Upload directories as release assets.")
+    parser.add_argument("repo", help="Repository name, e.g., 'username/repo'")
+    parser.add_argument("token", help="GitHub Token")
+    args = parser.parse_args()
+
+    # Elasticsearch instance used for indexing
+    num_of_retries = 1
+    while True:
+        print(f"Trying to connect to ES. Connection retry:  {num_of_retries}")
+        es = Elasticsearch([os.environ['ES_HOST']], http_auth=(os.environ['ES_USER'], os.environ['ES_PASSWORD']))
+        if es.ping():
+            break
+        # Wait for 30 seconds and try again if connection fails
+        if 10 == num_of_retries:
+            # Exit if it fails 10 times, something is wrong with the server
+            raise ValueError("Connection to ES failed!")
+        num_of_retries += 1
+
+        time.sleep(30)
+
+    # Now index the new release
+    index_release_to_elasticsearch(
+        es, os.environ['ES_INDEX'],
+        fetch_release_details(args.repo, args.token),
+        args.token
+    )

scripts/package.py

@@ -1,6 +1,4 @@
-import os, sys, py7zr, requests, time
-
-from elasticsearch import Elasticsearch
+import os, re, py7zr, requests, argparse, json
 
 def find_manifest_folder(base_dir):
     """Find the folder containing 'manifest.json'."""
@@ -13,12 +11,7 @@ def create_7z_archive(version, source_folder, archive_path):
     """Create a .7z archive from a source folder with a specific folder structure, excluding the .github folder."""
     with py7zr.SevenZipFile(archive_path, 'w') as archive:
         for root, dirs, files in os.walk(source_folder):
-            relative_path = os.path.relpath(root, source_folder)
-            if '.github' in relative_path.split(os.sep):
-                continue
-            if '.git' in relative_path.split(os.sep):
-                continue
-            if 'scripts' in relative_path.split(os.sep):
+            if re.search(r'(\.git)|(scripts)|(templates)', os.path.relpath(root, source_folder)):
                 continue
             for file in files:
                 file_path = os.path.join(root, file)
@@ -27,6 +20,13 @@ def create_7z_archive(version, source_folder, archive_path):
                     continue
                 archive.write(file_path, os.path.join(version, 'src', os.path.relpath(file_path, source_folder)))
 
+def create_template_archive(source_folder, archive_path):
+    """Create a .7z archive from a source folder with a specific folder structure."""
+    with py7zr.SevenZipFile(archive_path, 'w') as archive:
+        os.chdir(source_folder)
+        archive.writeall('./')
+        os.chdir('..')
+
 def upload_asset_to_release(repo, release_id, asset_path, token):
     """Upload an asset to a specific GitHub release."""
     url = f'https://uploads.github.com/repos/{repo}/releases/{release_id}/assets?name={os.path.basename(asset_path)}'
@@ -39,7 +39,6 @@ def upload_asset_to_release(repo, release_id, asset_path, token):
         response.raise_for_status()
         print(f'Uploaded asset: {os.path.basename(asset_path)} to release ID: {release_id}')
         return response.json()
-    return None
 
 def get_release_id(repo, tag_name, token):
     """Get the release ID for a given tag name."""
@@ -50,53 +49,36 @@ def get_release_id(repo, tag_name, token):
     return response.json()['id']
 
 if __name__ == '__main__':
-    token = sys.argv[1]
-    repo = sys.argv[2]
-    tag_name = sys.argv[3]
-
-    # Elasticsearch details
-    num_of_retries = 1
-    while True:
-        es = Elasticsearch([os.environ['ES_HOST']])
-        if es.ping():
-            break
-        # Wait for 30 seconds and try again if connection fails
-        if 10 == num_of_retries:
-            # Exit if it fails 10 times, something is wrong with the server
-            raise ValueError("Connection to ES failed!")
-        num_of_retries += 1
-        time.sleep(30)
-    index_name = os.environ['ES_INDEX']
+    parser = argparse.ArgumentParser(description="Upload directories as release assets.")
+    parser.add_argument("token", help="GitHub Token")
+    parser.add_argument("repo", help="Repository name, e.g., 'username/repo'")
+    args = parser.parse_args()
 
     # Assuming the repository is checked out at the root directory
     repo_dir = os.getcwd()
-    version = tag_name.split('-')[1]
-
     manifest_folder = find_manifest_folder(repo_dir)
+    version = json.load(open(os.path.join(manifest_folder ,'manifest.json')))['sdk-version']
+
     if manifest_folder:
         archive_path = os.path.join(repo_dir, 'mikrosdk.7z')
         print('Creating archive: %s' % archive_path)
         create_7z_archive(version, repo_dir, archive_path)
         print('Archive created successfully: %s' % archive_path)
 
         # Get the release ID and upload the asset
-        release_id = get_release_id(repo, tag_name, token)
-        upload_result = upload_asset_to_release(repo, release_id, archive_path, token)
-        doc = {
-            'name': upload_result['name'],
-            'display_name': "mikroSDK",
-            'author': 'MIKROE',
-            'hidden': False,
-            'type': 'sdk',
-            'version': tag_name.replace('mikroSDK-', ''),
-            'created_at' : upload_result['created_at'],
-            'updated_at' : upload_result['updated_at'],
-            'category': 'Software Development Kit',
-            'download_link': upload_result['browser_download_url'],  # Adjust as needed for actual URL
-            'package_changed': True
-        }
-        resp = es.index(index=index_name, doc_type='mikrosdk_package', id='mikroSDK', body=doc)
-        print(f"ES RESPONSE: {resp}")
+        release_id = get_release_id(args.repo, f'mikroSDK-{version}', args.token)
+        upload_result = upload_asset_to_release(args.repo, release_id, archive_path, args.token)
+
+        print('Asset "%s" uploaded successfully to release ID: %s' % ('mikrosdk', release_id))
+
+    if os.path.exists(os.path.join(repo_dir, 'templates')):
+        archive_path = os.path.join(repo_dir, 'templates.7z')
+        print('Creating archive: %s' % archive_path)
+        create_template_archive('templates', archive_path)
+        print('Archive created successfully: %s' % archive_path)
+
+        # Get the release ID and upload the asset
+        release_id = get_release_id(args.repo, f'mikroSDK-{version}', args.token)
+        upload_result = upload_asset_to_release(args.repo, release_id, archive_path, args.token)
 
-        print('Asset details: \n%s' % doc)
-        print('Asset uploaded successfully to release ID: %s' % release_id)
+        print('Asset "%s" uploaded successfully to release ID: %s' % ('templates', release_id))

scripts/support.py

@@ -0,0 +1,79 @@
+import requests, py7zr, zipfile, io, os, json
+
+def get_previous_release(releases, prerelases=None):
+    ''' Fetch the previously released version '''
+    for counter, release in enumerate(releases):
+        if not release['draft']:
+            if prerelases:
+                if release['prerelease']:
+                    continue
+            if counter + 1 < len(releases):
+                return releases[counter + 1]
+            else:
+                return None
+    return None
+
+def get_latest_release(releases):
+    ''' Fetch the latest released version '''
+    return next((release for release in releases if not release['prerelease'] and not release['draft']), None)
+
+def determine_archive_type(byte_stream):
+    '''
+    Implement logic to determine the archive type, e.g., by file extension or magic number
+    For simplicity, let's assume byte_stream has a 'name' attribute (e.g., a file-like object)
+    '''
+    byte_stream.seek(0)
+    signature = byte_stream.read(4)
+    byte_stream.seek(0)
+    if signature == b'PK\x03\x04':  # ZIP magic number, it is what it is
+        return 'zip'
+    else:
+        return '7z'
+
+def extract_archive_from_url(url, destination, token):
+    """
+    Extract the contents of an archive (7z or zip) from a URL directly
+    in memory, without downloading the file.
+    """
+    print(f"Download link: {url}")
+    headers = {
+        'Authorization': f'token {token}',
+        'Accept': 'application/octet-stream'
+    }
+    if 'github' in url:
+        response = requests.get(url, headers=headers, stream=True)
+    else:
+        response = requests.get(url, stream=True)
+
+    response.raise_for_status()
+
+    if response.status_code == 200: ## Response OK?
+        with io.BytesIO() as byte_stream:
+
+            for chunk in response.iter_content(chunk_size=8192):
+                byte_stream.write(chunk)
+
+            byte_stream.seek(0)
+
+            archive_type = determine_archive_type(byte_stream)
+
+            if archive_type == '7z':
+                with py7zr.SevenZipFile(byte_stream, mode='r') as archive:
+                    archive.extractall(path=destination)
+            elif archive_type == 'zip':
+                with zipfile.ZipFile(byte_stream, mode='r') as archive:
+                    for info in archive.infolist():
+                        archive.extract(info, path=destination)
+            else:
+                raise ValueError("Unsupported archive type")
+    else:
+        raise Exception(f"Failed to download file: status code {response.status_code}")
+
+def fetch_version_from_asset(asset_path):
+    for root, subdirs, files in os.walk(asset_path):
+        for file_name in files:
+            if 'manifest.json' == file_name:
+                return json.load(open(os.path.join(root, file_name)))['sdk-version']
+            else:
+                continue
+    return None