Integrate mypy and yamlfix, fix Conda dependency resolution, add release workflow

.github/workflows/ci.yml

@@ -1,34 +1,26 @@
+---
 name: VSCode Extension CI
-
 on: [push, pull_request]
-
 jobs:
   build-and-test:
     runs-on: ubuntu-latest
-
     steps:
       - name: Checkout Repository
         uses: actions/checkout@v2
-
       - name: Install Node.js
         uses: actions/setup-node@v2
         with:
-          node-version: '18.x'
-
+          node-version: 18.x
       - name: Install dependencies
         run: |
           npm install
-          pip install autoflake isort black ruff
-
-      - name: Build VSCode Extension
-        run: npm run compile
-
-      - name: Run Format Script for Python & TypeScript Files
+          pip install autoflake isort black ruff yamlfix mypy
+      - name: Format Python, TypeScript, and YAML Files
         run: ./scripts/format.sh
-
-      - name: Run Lint Script for Python & TypeScript Files
+      - name: Lint Python, TypeScript, and YAML files
         run: ./scripts/lint.sh
-
+      - name: Compile VSCode Extension
+        run: npm run compile
       - name: Run headless test
         uses: coactions/setup-xvfb@v1
         with:

.github/workflows/release.yml

@@ -0,0 +1,49 @@
+---
+name: Publish VSCode Extension
+
+on:
+  release:
+    types: [created]
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v2
+
+      - name: Install Node.js
+        uses: actions/setup-node@v2
+        with:
+          node-version: 18.x
+          cache: 'npm'
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Build Extension
+        run: npm run package
+
+      - name: Package Extension
+        run: npm run vsce-package
+
+      - name: Publish Extension
+        if: success() && startsWith(github.ref, 'refs/tags/')
+        run: npm run deploy
+        env:
+          VSCE_PAT: ${{ secrets.VSCE_PAT }}
+
+      - name: Generate Changelog
+        if: success() && startsWith(github.ref, 'refs/tags/')
+        run: |
+          git log $(git describe --tags --abbrev=0)..HEAD --oneline > CHANGELOG.txt
+          cat CHANGELOG.txt
+
+      - name: Create GitHub Release
+        if: success() && startsWith(github.ref, 'refs/tags/')
+        uses: ncipollo/release-action@v1
+        with:
+          artifacts: 'zenml.vsix'
+          bodyFile: 'CHANGELOG.txt'
+          tag: ${{ github.ref_name }}
+          token: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -28,6 +28,9 @@ build/
 # Output of 'npm pack'
 *.tgz
 
+# mypy
+.mypy_cache/
+
 # From vscode-python-tools-extension-template
 *.vsix
 .venv/
@@ -37,4 +40,3 @@ bundled/libs/
 **/__pycache__
 **/.pytest_cache
 **/.vs
-

RELEASE.md

@@ -0,0 +1,62 @@
+# Release Process
+
+This document describes the process of publishing releases for our VS Code extension and provides an explanation of the GitHub Actions workflow file.
+
+## Overview
+
+The release process is automated using GitHub Actions. When a new release is created on GitHub, it triggers the release workflow defined in `.github/workflows/release.yml`. The workflow performs the following steps:
+
+1. Checks out the repository.
+2. Installs Node.js and the required dependencies.
+3. Builds the extension using webpack.
+4. Packages the extension into a `.vsix` file.
+5. Publishes the extension to the Visual Studio Code Marketplace.
+6. Generates a changelog based on the commit messages.
+7. Creates a GitHub release with the packaged extension file as an artifact and the changelog.
+
+## Prerequisites
+
+Before creating a release, ensure that:
+
+- The extension is properly configured and builds successfully.
+- The Personal Access Token (PAT) is set as a repository secret named `VSCE_PAT`.
+
+## Creating a Release
+
+To create a new release:
+
+1. Go to the GitHub repository page.
+2. Click on the "Releases" tab.
+3. Click on the "Draft a new release" button.
+4. Enter the tag version for the release (e.g., `v1.0.0`).
+5. Set the release title and description.
+6. Choose the appropriate release type (e.g., pre-release or stable release).
+7. Click on the "Publish release" button.
+
+Creating the release will trigger the release workflow automatically.
+
+## Workflow File Explanation
+
+The release workflow is defined in `.github/workflows/release.yml`. Here's an explanation of each step in the workflow:
+
+1. **Checkout Repository**: This step checks out the repository using the `actions/checkout@v2` action.
+
+2. **Install Node.js**: This step sets up Node.js using the `actions/setup-node@v2` action. It specifies the Node.js version and enables caching of npm dependencies.
+
+3. **Install dependencies**: This step runs `npm ci` to install the project dependencies.
+
+4. **Build Extension**: This step runs `npm run package` to build the extension using webpack.
+
+5. **Package Extension**: This step runs `npm run vsce-package` to package the extension into a `.vsix` file named `zenml.vsix`.
+
+6. **Publish Extension**: This step runs `npm run deploy` to publish the extension to the Visual Studio Code Marketplace. It uses the `VSCE_PAT` secret for authentication. This step only runs if the previous steps succeeded and the workflow was triggered by a new tag push.
+
+7. **Generate Changelog**: This step generates a changelog by running `git log` to retrieve the commit messages between the latest tag and the current commit. The changelog is saved in a file named `CHANGELOG.txt`. This step only runs if the previous steps succeeded and the workflow was triggered by a new tag push.
+
+8. **Create GitHub Release**: This step uses the `ncipollo/release-action@v1` action to create a GitHub release. It attaches the `zenml.vsix` file as an artifact, includes the changelog, and sets the release tag based on the pushed tag. This step only runs if the previous steps succeeded and the workflow was triggered by a new tag push.
+
+## Conclusion
+
+The provided GitHub Actions workflow automates the publishing of the ZenML VSCode extension. The workflow ensures that the extension is built, packaged, published to the marketplace, and a GitHub release is created with the necessary artifacts and changelog.
+
+Remember to keep the extension code up to date, maintain the required dependencies, and test the extension thoroughly before creating a release.

bundled/tool/lsp_jsonrpc.py

@@ -22,7 +22,7 @@
 import threading
 import uuid
 from concurrent.futures import ThreadPoolExecutor
-from typing import BinaryIO, Dict, Optional, Sequence, Union
+from typing import BinaryIO, Dict, Optional, Sequence, Union, cast
 
 CONTENT_LENGTH = "Content-Length: "
 RUNNER_SCRIPT = str(pathlib.Path(__file__).parent / "lsp_runner.py")
@@ -60,7 +60,10 @@ def write(self, data):
         with self._lock:
             content = json.dumps(data)
             length = len(content.encode("utf-8"))
-            self._writer.write(f"{CONTENT_LENGTH}{length}\r\n\r\n{content}".encode("utf-8"))
+            self._writer.write(f"{CONTENT_LENGTH}{length}\r\n\r\n{content}")
+            # self._writer.write(
+            #     f"{CONTENT_LENGTH}{length}\r\n\r\n{content}".encode("utf-8")
+            # )
             self._writer.flush()
 
 
@@ -128,7 +131,9 @@ def receive_data(self):
 
 def create_json_rpc(readable: BinaryIO, writable: BinaryIO) -> JsonRpc:
     """Creates JSON-RPC wrapper for the readable and writable streams."""
-    return JsonRpc(readable, writable)
+    text_readable = io.TextIOWrapper(readable, encoding="utf-8")
+    text_writable = io.TextIOWrapper(writable, encoding="utf-8")
+    return JsonRpc(text_readable, text_writable)
 
 
 class ProcessManager:
@@ -159,8 +164,13 @@ def start_process(self, workspace: str, args: Sequence[str], cwd: str) -> None:
             stdout=subprocess.PIPE,
             stdin=subprocess.PIPE,
         )
+
+        # Use cast to assure mypy that stdout and stdin are not None
+        stdout = cast(BinaryIO, proc.stdout)
+        stdin = cast(BinaryIO, proc.stdin)
+
         self._processes[workspace] = proc
-        self._rpc[workspace] = create_json_rpc(proc.stdout, proc.stdin)
+        self._rpc[workspace] = create_json_rpc(stdout, stdin)
 
         def _monitor_process():
             proc.wait()
@@ -224,7 +234,7 @@ def run_over_json_rpc(
     argv: Sequence[str],
     use_stdin: bool,
     cwd: str,
-    source: str = None,
+    source: Optional[str] = None,
 ) -> RpcRunResult:
     """Uses JSON-RPC to execute a command."""
     rpc: Union[JsonRpc, None] = get_or_start_json_rpc(workspace, interpreter, cwd)

bundled/tool/lsp_server.py

@@ -17,7 +17,7 @@
 import os
 import pathlib
 import sys
-from typing import Any, Dict, Optional, Tuple
+from typing import Any, Dict, List, Optional, Tuple
 
 from constants import TOOL_DISPLAY_NAME, TOOL_MODULE_NAME, ZENML_CLIENT_INITIALIZED
 
@@ -50,8 +50,8 @@ def update_sys_path(path_to_add: str, strategy: str) -> None:
 from lsp_zenml import ZenLanguageServer  # noqa: E402
 from pygls import uris, workspace  # noqa: E402
 
-WORKSPACE_SETTINGS = {}
-GLOBAL_SETTINGS = {}
+WORKSPACE_SETTINGS: Dict[str, Any] = {}
+GLOBAL_SETTINGS: Dict[str, Any] = {}
 RUNNER = pathlib.Path(__file__).parent / "lsp_runner.py"
 
 MAX_WORKERS = 5
@@ -63,8 +63,8 @@ def update_sys_path(path_to_add: str, strategy: str) -> None:
 # **********************************************************
 TOOL_MODULE = TOOL_MODULE_NAME
 TOOL_DISPLAY = TOOL_DISPLAY_NAME
-# Default arguments always passed to zenml.
-TOOL_ARGS = []
+# Default arguments always passed to zenml. (Not currently used)
+TOOL_ARGS: List[str] = []
 # Versions of zenml found by workspace
 VERSION_LOOKUP: Dict[str, Tuple[int, int, int]] = {}
 
@@ -81,13 +81,20 @@ async def initialize(params: lsp.InitializeParams) -> None:
     paths = "\r\n   ".join(sys.path)
     log_to_output(f"sys.path used to run Server:\r\n   {paths}")
 
-    GLOBAL_SETTINGS.update(**params.initialization_options.get("globalSettings", {}))
+    # Check if initialization_options is a dictionary and update GLOBAL_SETTINGS safely
+    if isinstance(params.initialization_options, dict):
+        global_settings = params.initialization_options.get("globalSettings", {})
+        if isinstance(global_settings, dict):
+            GLOBAL_SETTINGS.update(**global_settings)
+
+        # Safely access 'settings' from initialization_options if present
+        settings = params.initialization_options.get("settings")
+        if settings is not None:
+            _update_workspace_settings(settings)
+            log_to_output(
+                f"Settings used to run Server:\r\n{json.dumps(settings, indent=4, ensure_ascii=False)}\r\n"
+            )
 
-    settings = params.initialization_options["settings"]
-    _update_workspace_settings(settings)
-    log_to_output(
-        f"Settings used to run Server:\r\n{json.dumps(settings, indent=4, ensure_ascii=False)}\r\n"
-    )
     log_to_output(
         f"Global settings:\r\n{json.dumps(GLOBAL_SETTINGS, indent=4, ensure_ascii=False)}\r\n"
     )

bundled/tool/lsp_utils.py

@@ -23,17 +23,19 @@
 import subprocess
 import sys
 import threading
-from typing import Any, Callable, List, Sequence, Tuple, Union
+from typing import Any, Callable, List, Optional, Sequence, Tuple, Union
 
 # Save the working directory used when loading this module
 SERVER_CWD = os.getcwd()
 CWD_LOCK = threading.Lock()
 
 
-def as_list(content: Union[Any, List[Any], Tuple[Any]]) -> Union[List[Any], Tuple[Any]]:
+def as_list(content: Union[Any, List[Any], Tuple[Any, ...]]) -> List[Any]:
     """Ensures we always get a list"""
-    if isinstance(content, (list, tuple)):
+    if isinstance(content, list):
         return content
+    elif isinstance(content, tuple):
+        return list(content)  # Convert tuple to list
     return [content]
 
 
@@ -118,7 +120,9 @@ def change_cwd(new_cwd):
     os.chdir(SERVER_CWD)
 
 
-def _run_module(module: str, argv: Sequence[str], use_stdin: bool, source: str = None) -> RunResult:
+def _run_module(
+    module: str, argv: Sequence[str], use_stdin: bool, source: Optional[str] = None
+) -> RunResult:
     """Runs as a module."""
     str_output = CustomIO("<stdout>", encoding="utf-8")
     str_error = CustomIO("<stderr>", encoding="utf-8")
@@ -142,7 +146,11 @@ def _run_module(module: str, argv: Sequence[str], use_stdin: bool, source: str =
 
 
 def run_module(
-    module: str, argv: Sequence[str], use_stdin: bool, cwd: str, source: str = None
+    module: str,
+    argv: Sequence[str],
+    use_stdin: bool,
+    cwd: str,
+    source: Optional[str] = None,
 ) -> RunResult:
     """Runs as a module."""
     with CWD_LOCK:
@@ -152,7 +160,9 @@ def run_module(
             return _run_module(module, argv, use_stdin, source)
 
 
-def run_path(argv: Sequence[str], use_stdin: bool, cwd: str, source: str = None) -> RunResult:
+def run_path(
+    argv: Sequence[str], use_stdin: bool, cwd: str, source: Optional[str] = None
+) -> RunResult:
     """Runs as an executable."""
     if use_stdin:
         with subprocess.Popen(
@@ -181,7 +191,7 @@ def run_api(
     argv: Sequence[str],
     use_stdin: bool,
     cwd: str,
-    source: str = None,
+    source: Optional[str] = None,
 ) -> RunResult:
     """Run a API."""
     with CWD_LOCK:
@@ -195,7 +205,7 @@ def _run_api(
     callback: Callable[[Sequence[str], CustomIO, CustomIO, CustomIO | None], None],
     argv: Sequence[str],
     use_stdin: bool,
-    source: str = None,
+    source: Optional[str] = None,
 ) -> RunResult:
     str_output = CustomIO("<stdout>", encoding="utf-8")
     str_error = CustomIO("<stderr>", encoding="utf-8")
@@ -211,7 +221,7 @@ def _run_api(
                             str_input.seek(0)
                             callback(argv, str_output, str_error, str_input)
                     else:
-                        callback(argv, str_output, str_error)
+                        callback(argv, str_output, str_error, None)
     except SystemExit:
         pass
 

mypy.ini

@@ -0,0 +1,9 @@
+[mypy]
+python_version = 3.8
+ignore_missing_imports = True
+check_untyped_defs = True
+disallow_untyped_defs = False
+warn_unused_ignores = True
+
+[mypy-bundled.tool.*]
+ignore_errors = False