diff --git a/CHANGELOG.md b/CHANGELOG.md index d79991f..6f07f81 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Release Notes +**v2.0.0:** +- added integration with [official Databricks extensions](https://marketplace.visualstudio.com/items?itemName=databricks.databricks) + - new connection manager [Databricks Extensions](README.md/#setup-and-configuration-databricks-extension-connection-manager) + - derive cluster for [SQL Browser](README.md/#sql-browser) + - change cluster using [Cluster Manager](README.md/#cluster-manager) + - automatically create a [Notebook Kernel](README.md/#notebook-kernel) for the configured cluster + - added File System `wsfs:/` to replace `dbws:/` in the future (currently both are still supported) + **v1.5.0:** - added support for [Widgets](README.md/#widgets) when running Notebooks diff --git a/README.md b/README.md index a81051b..a4d3095 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# VSCode Extension for Databricks +# Databricks Power Tools for VSCode [![Version](https://vsmarketplacebadges.dev/version/paiqo.databricks-vscode.svg?color=blue&style=?style=for-the-badge&logo=visual-studio-code)](https://marketplace.visualstudio.com/items?itemName=paiqo.databricks-vscode) [![Installs](https://vsmarketplacebadges.dev/installs/paiqo.databricks-vscode.svg?color=yellow)](https://marketplace.visualstudio.com/items?itemName=paiqo.databricks-vscode) [![Downloads](https://vsmarketplacebadges.dev/downloads/paiqo.databricks-vscode.svg?color=yellow)](https://marketplace.visualstudio.com/items?itemName=paiqo.databricks-vscode) @@ -6,7 +6,7 @@ ![Databricks-VSCode](/images/Databricks-VSCode.jpg?raw=true "Databricks-VSCode") -This is a Visual Studio Code extension that allows you to work with Databricks locally from VSCode in an efficient way, having everything you need integrated into VS Code - see [Features](#features). It allows you to manage and execute your notebooks, start/stop clusters, execute jobs and much more! +This is a Visual Studio Code extension that allows you to work with Databricks locally from VSCode in an efficient way, having everything you need integrated into VS Code - see [Features](#features). It allows you to execute your notebooks, start/stop clusters, execute jobs and much more! The extensions can be downloaded from the official Visual Studio Code extension gallery: [Databricks VSCode](https://marketplace.visualstudio.com/items?itemName=paiqo.databricks-vscode) @@ -36,6 +36,7 @@ The extensions can be downloaded from the official Visual Studio Code extension - control how notebooks are downloaded (Jupyter notebook, source code, ...) - various other settings - Load Databricks directly from your Azure Account + - Leverage connections configured by the official Databricks VSCode extension - [SQL / Data Browser](#sql-browser) - Browse availabl SQL objects from the Databricks metastore - databases, tables and views, columns, ... @@ -65,11 +66,14 @@ Alternatively it can also be downloaded the `.vsix` directly from the VS Code m Preview-Versions might also be available via github [Releases](https://github.com/paiqo/Databricks-VSCode/releases) from this repository. -# Setup and Configuration (VSCode Connection Manager) +# Setup and Configuration The configuration happens directly via VS Code by simply [opening the settings](https://code.visualstudio.com/docs/getstarted/settings#_creating-user-and-workspace-settings) Then either search for "Databricks" or expand Extensions -> Databricks. -The settings themselves are very well described and it should be easy for you to populate them. Also, not all of them are mandatory! Some of the optional settings are experimental or still work in progress. -To configure multiple Databricks Connections/workspaces, you need to use the JSON editor and add them to `databricks.connections`: +The most important setting to start with is definitly `databricks.connectionManager` as it defines how you manage your connections. There are a couple of differnt options which are described further down below. +All the settings themselves are very well described and it should be easy for you to populate them. Also, not all of them are mandatory, and depend a lot on the connection manager that you have chosen.Some of the optional settings are experimental or still work in progress. + +# Setup and Configuration (VSCode Connection Manager) +Using `VSCode Settings` as your connection manager allows you to define and manage your connections directly from within VSCode via regular VSCode settings. It is recommended to use workspace settings over user settings here as it might get confusing otherwise. The default connection can be configured directly via the settings UI using the `databricks.connection.default.*` settings. To configure multiple Databricks Connections/workspaces, you need to use the JSON editor and add them to `databricks.connections`: ``` json ... @@ -169,6 +173,10 @@ The following Azure-specific settings exist and can be set in the workspace sett They are documented via VSCode settings documentation. +# Setup and Configuration (Databricks Extension Connection Manager) +This connection manager leverages the [official Databricks extensions](https://marketplace.visualstudio.com/items?itemName=databricks.databricks) to establish a connection with your Databricks workspace. It only supports a single connection hence the actual Connection Manager tab will be hidden for this connection manager. +It also derives the cluster automatically from the Databricks extensions to source the [SQL Browser](#sql-browser) but also allows you to change it directly from the [Cluster Manager](#cluster-manager) using the `Attach cluster` command from the context menu! + # Connection Manager ![Connection Manager](/images/ConnectionManager.jpg?raw=true "Connection Manager") @@ -176,6 +184,7 @@ The extension supports various connection managers and the list can be easily ex - [VSCode Settings](#setup-and-configuration-vscode-connection-manager) - [Databricks CLI](#setup-and-configuration-databricks-cli-connection-manager) - [Azure](#setup-and-configuration-azure-connection-manager) +- [Databricks Extensions](#setup-and-configuration-databricks-extensions-connection-manager) - `Manual` where you are simply prompted to enter connection information at the start of your session. You can specify the one to use by setting the VSCode setting `databricks.connectionManager`. @@ -215,10 +224,12 @@ For better visualization of tabluar results this extension includes a dependency Notebook Kernels also support other features like [Files in Repo](https://docs.databricks.com/_static/notebooks/files-in-repos.html) to build libraries within your repo, [_sqldf](https://docs.databricks.com/notebooks/notebooks-use.html#explore-sql-cell-results-in-python-notebooks-natively-using-python) to expose results of SQL cells to Python/Pyspark, `%run` to run other notebooks inline with the current notebook and also [dbutils.notebook.run()](https://docs.databricks.com/dev-tools/databricks-utils.html#notebook-utility-dbutilsnotebook). -Whenever a notebook is opened from either the local sync folder or via the [Virtual File System](#file-system-integration) using `dbws:/` URI, the Databricks notebook kernels are the preferred ones and should appear at the top of the list when you select a kernel. +Whenever a notebook is opened from either the local sync folder or via the [Virtual File System](#file-system-integration) using `wsfs:/` URI, the Databricks notebook kernels are the preferred ones and should appear at the top of the list when you select a kernel. + +If you are using the [Databricks Extension Connection Manager](#setup-and-configuration-databricks-extension-connection-manager) we will also create a generic notebook kernel for you which used the configured cluster. ## Execution Modes -We distinguish between Live-execution and Offline-execution. In Live-execution mode, files are opened directly from Databricks by mounting the Databricks Workspace into your VSCode Workspace using `dbws:/` URI scheme. In this mode there is no intermediate local copy but you work directly against the Databricks Workspace. Everything you run must already exist online in the Databricks Workspace. +We distinguish between Live-execution and Offline-execution. In Live-execution mode, files are opened directly from Databricks by mounting the Databricks Workspace into your VSCode Workspace using `wsfs:/` URI scheme. In this mode there is no intermediate local copy but you work directly against the Databricks Workspace. Everything you run must already exist online in the Databricks Workspace. This is slightly different in Offline-execution where all files you want to work with need to be synced locally first using the [Workspace Manager](#workspace-manager). This is especially important when it comes `%run` which behaves slightly differntly compared to Live-execution mode. `%run` in Offline-execution runs the code from your local file instead of the code that exists in Dtabricks online! Other commands like `dbutils.notebook.run()` always use the code thats currently online so if you have changed the refernced notebook locally, you have to upload it first. This is simply because we cannot easily replicate the behavior of `dbutils.notebook.run()` locally! @@ -245,7 +256,8 @@ You want to upload a local notebook to the Databricks workspace? Simply drag&dro You want to download a file from DBFS? Simply drag&drop it! There are two virtual file systems that come with this extension: -- `dbws:/` to access your notebook from the DAtabricks workspace +- `wsfs:/` to access your notebook from the DAtabricks workspace +- `dbws:/` (LEGACY) - to be replaced by `wsfs:/` in the long term - `dbfs:/` to access files on the Databricks File System (DBFS) - similar to the [DBFS Browser](#dbfs-browser) # SQL Browser diff --git a/package-lock.json b/package-lock.json index a45e914..761bd15 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "databricks-vscode", - "version": "1.4.0", + "version": "2.0.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "databricks-vscode", - "version": "1.4.0", + "version": "2.0.0", "license": "GPL-3.0-or-later", "devDependencies": { "@types/glob": "^7.2.0", @@ -15,7 +15,7 @@ "@types/rimraf": "^2.0.5", "@types/vscode": "^1.68.0", "@vscode/test-web": "*", - "@vscode/vsce": "^2.15.0", + "@vscode/vsce": "^2.18.0", "eslint": "^8.19.0", "glob": "^8.0.3", "mocha": "^10.0.0", @@ -25,7 +25,7 @@ "process": "^0.11.10", "ts-loader": "^9.3.1", "typescript": "^4.7.4", - "webpack": "^5.74.0", + "webpack": "^5.76.0", "webpack-cli": "^4.10.0" }, "engines": { @@ -331,9 +331,9 @@ } }, "node_modules/@vscode/vsce": { - "version": "2.15.0", - "resolved": "https://registry.npmjs.org/@vscode/vsce/-/vsce-2.15.0.tgz", - "integrity": "sha512-c+qS5KSX4jO3RuGqeNQHqci4+WrcmLxHAwiWTR3PDR6wXzV1fQJxybueUOojXcqvsJR3W2AeROrpf+302ZkTfg==", + "version": "2.18.0", + "resolved": "https://registry.npmjs.org/@vscode/vsce/-/vsce-2.18.0.tgz", + "integrity": "sha512-tUA3XoKx5xjoi3EDcngk0VUYMhvfXLhS4s7CntpLPh1qtLYtgSCexTIMUHkCy6MqyozRW98bdW3a2yHPEADRnQ==", "dev": true, "dependencies": { "azure-devops-node-api": "^11.0.1", @@ -342,7 +342,7 @@ "commander": "^6.1.0", "glob": "^7.0.6", "hosted-git-info": "^4.0.2", - "keytar": "^7.7.0", + "jsonc-parser": "^3.2.0", "leven": "^3.1.0", "markdown-it": "^12.3.2", "mime": "^1.3.4", @@ -362,6 +362,9 @@ }, "engines": { "node": ">= 14" + }, + "optionalDependencies": { + "keytar": "^7.7.0" } }, "node_modules/@vscode/vsce/node_modules/ansi-styles": { @@ -1146,7 +1149,8 @@ "version": "1.1.4", "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", - "dev": true + "dev": true, + "optional": true }, "node_modules/chrome-trace-event": { "version": "1.0.3", @@ -1372,6 +1376,7 @@ "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==", "dev": true, + "optional": true, "dependencies": { "mimic-response": "^3.1.0" }, @@ -1483,6 +1488,7 @@ "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", "dev": true, + "optional": true, "engines": { "node": ">=4.0.0" } @@ -1523,6 +1529,7 @@ "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.1.tgz", "integrity": "sha512-463v3ZeIrcWtdgIg6vI6XUncguvr2TnGl4SzDXinkt9mSLpBJKXT3mW6xT3VQdDN11+WVs29pgvivTc4Lp8v+w==", "dev": true, + "optional": true, "engines": { "node": ">=8" } @@ -1887,6 +1894,7 @@ "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==", "dev": true, + "optional": true, "engines": { "node": ">=6" } @@ -2128,7 +2136,8 @@ "version": "0.0.0", "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==", - "dev": true + "dev": true, + "optional": true }, "node_modules/glob": { "version": "8.0.3", @@ -2486,7 +2495,8 @@ "version": "1.3.8", "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", - "dev": true + "dev": true, + "optional": true }, "node_modules/interpret": { "version": "2.2.0", @@ -2718,6 +2728,12 @@ "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==", "dev": true }, + "node_modules/jsonc-parser": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/jsonc-parser/-/jsonc-parser-3.2.0.tgz", + "integrity": "sha512-gfFQZrcTc8CnKXp6Y4/CBT3fTc0OVuDofpre4aEeEpSBPV5X5v4+Vmx+8snU7RLPrNHPKSgLxGo9YuQzz20o+w==", + "dev": true + }, "node_modules/keygrip": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/keygrip/-/keygrip-1.1.0.tgz", @@ -2736,6 +2752,7 @@ "integrity": "sha512-VPD8mtVtm5JNtA2AErl6Chp06JBfy7diFQ7TQQhdpWOl6MrCRB+eRbvAZUsbGQS9kiMq0coJsy0W0vHpDCkWsQ==", "dev": true, "hasInstallScript": true, + "optional": true, "dependencies": { "node-addon-api": "^4.3.0", "prebuild-install": "^7.0.1" @@ -3145,6 +3162,7 @@ "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==", "dev": true, + "optional": true, "engines": { "node": ">=10" }, @@ -3177,7 +3195,8 @@ "version": "0.5.3", "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==", - "dev": true + "dev": true, + "optional": true }, "node_modules/mocha": { "version": "10.1.0", @@ -3364,7 +3383,8 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-1.0.2.tgz", "integrity": "sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg==", - "dev": true + "dev": true, + "optional": true }, "node_modules/native-ext-loader": { "version": "2.3.0", @@ -3398,6 +3418,7 @@ "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.28.0.tgz", "integrity": "sha512-fRlDb4I0eLcQeUvGq7IY3xHrSb0c9ummdvDSYWfT9+LKP+3jCKw/tKoqaM7r1BAoiAC6GtwyjaGnOz6B3OtF+A==", "dev": true, + "optional": true, "dependencies": { "semver": "^7.3.5" }, @@ -3410,6 +3431,7 @@ "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz", "integrity": "sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==", "dev": true, + "optional": true, "dependencies": { "lru-cache": "^6.0.0" }, @@ -3424,7 +3446,8 @@ "version": "4.3.0", "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-4.3.0.tgz", "integrity": "sha512-73sE9+3UaLYYFmDsFZnqCInzPyh3MqIwZO9cw58yIqAZhONrrabrYyYe3TuIqtIiOuTXVhsGau8hcrhhwSsDIQ==", - "dev": true + "dev": true, + "optional": true }, "node_modules/node-domexception": { "version": "1.0.0", @@ -3851,6 +3874,7 @@ "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.1.tgz", "integrity": "sha512-jAXscXWMcCK8GgCoHOfIr0ODh5ai8mj63L2nWrjuAgXE6tDyYGnx4/8o/rCgU+B4JSyZBKbeZqzhtwtC3ovxjw==", "dev": true, + "optional": true, "dependencies": { "detect-libc": "^2.0.0", "expand-template": "^2.0.3", @@ -3901,6 +3925,7 @@ "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", "dev": true, + "optional": true, "dependencies": { "end-of-stream": "^1.1.0", "once": "^1.3.1" @@ -3964,6 +3989,7 @@ "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", "dev": true, + "optional": true, "dependencies": { "deep-extend": "^0.6.0", "ini": "~1.3.0", @@ -3979,6 +4005,7 @@ "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", "dev": true, + "optional": true, "engines": { "node": ">=0.10.0" } @@ -4380,7 +4407,8 @@ "type": "consulting", "url": "https://feross.org/support" } - ] + ], + "optional": true }, "node_modules/simple-get": { "version": "4.0.1", @@ -4401,6 +4429,7 @@ "url": "https://feross.org/support" } ], + "optional": true, "dependencies": { "decompress-response": "^6.0.0", "once": "^1.3.1", @@ -4535,6 +4564,7 @@ "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.1.tgz", "integrity": "sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==", "dev": true, + "optional": true, "dependencies": { "chownr": "^1.1.1", "mkdirp-classic": "^0.5.2", @@ -4547,6 +4577,7 @@ "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", "dev": true, + "optional": true, "dependencies": { "buffer": "^5.5.0", "inherits": "^2.0.4", @@ -4558,6 +4589,7 @@ "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", "dev": true, + "optional": true, "dependencies": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", @@ -4572,6 +4604,7 @@ "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", "dev": true, + "optional": true, "dependencies": { "bl": "^4.0.3", "end-of-stream": "^1.4.1", @@ -4761,6 +4794,7 @@ "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", "integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==", "dev": true, + "optional": true, "dependencies": { "safe-buffer": "^5.0.1" }, @@ -4936,9 +4970,9 @@ } }, "node_modules/webpack": { - "version": "5.75.0", - "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.75.0.tgz", - "integrity": "sha512-piaIaoVJlqMsPtX/+3KTTO6jfvrSYgauFVdt8cr9LTHKmcq/AMd4mhzsiP7ZF/PGRNPGA8336jldh9l2Kt2ogQ==", + "version": "5.76.1", + "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.76.1.tgz", + "integrity": "sha512-4+YIK4Abzv8172/SGqObnUjaIHjLEuUasz9EwQj/9xmPPkYJy2Mh03Q/lJfSD3YLzbxy5FeTq5Uw0323Oh6SJQ==", "dev": true, "dependencies": { "@types/eslint-scope": "^3.7.3", @@ -5518,9 +5552,9 @@ } }, "@vscode/vsce": { - "version": "2.15.0", - "resolved": "https://registry.npmjs.org/@vscode/vsce/-/vsce-2.15.0.tgz", - "integrity": "sha512-c+qS5KSX4jO3RuGqeNQHqci4+WrcmLxHAwiWTR3PDR6wXzV1fQJxybueUOojXcqvsJR3W2AeROrpf+302ZkTfg==", + "version": "2.18.0", + "resolved": "https://registry.npmjs.org/@vscode/vsce/-/vsce-2.18.0.tgz", + "integrity": "sha512-tUA3XoKx5xjoi3EDcngk0VUYMhvfXLhS4s7CntpLPh1qtLYtgSCexTIMUHkCy6MqyozRW98bdW3a2yHPEADRnQ==", "dev": true, "requires": { "azure-devops-node-api": "^11.0.1", @@ -5529,6 +5563,7 @@ "commander": "^6.1.0", "glob": "^7.0.6", "hosted-git-info": "^4.0.2", + "jsonc-parser": "^3.2.0", "keytar": "^7.7.0", "leven": "^3.1.0", "markdown-it": "^12.3.2", @@ -6149,7 +6184,8 @@ "version": "1.1.4", "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", - "dev": true + "dev": true, + "optional": true }, "chrome-trace-event": { "version": "1.0.3", @@ -6321,6 +6357,7 @@ "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==", "dev": true, + "optional": true, "requires": { "mimic-response": "^3.1.0" } @@ -6408,7 +6445,8 @@ "version": "0.6.0", "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", - "dev": true + "dev": true, + "optional": true }, "deep-is": { "version": "0.1.4", @@ -6438,7 +6476,8 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.1.tgz", "integrity": "sha512-463v3ZeIrcWtdgIg6vI6XUncguvr2TnGl4SzDXinkt9mSLpBJKXT3mW6xT3VQdDN11+WVs29pgvivTc4Lp8v+w==", - "dev": true + "dev": true, + "optional": true }, "diff": { "version": "5.0.0", @@ -6702,7 +6741,8 @@ "version": "2.0.3", "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==", - "dev": true + "dev": true, + "optional": true }, "fast-deep-equal": { "version": "3.1.3", @@ -6879,7 +6919,8 @@ "version": "0.0.0", "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==", - "dev": true + "dev": true, + "optional": true }, "glob": { "version": "8.0.3", @@ -7136,7 +7177,8 @@ "version": "1.3.8", "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", - "dev": true + "dev": true, + "optional": true }, "interpret": { "version": "2.2.0", @@ -7310,6 +7352,12 @@ "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==", "dev": true }, + "jsonc-parser": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/jsonc-parser/-/jsonc-parser-3.2.0.tgz", + "integrity": "sha512-gfFQZrcTc8CnKXp6Y4/CBT3fTc0OVuDofpre4aEeEpSBPV5X5v4+Vmx+8snU7RLPrNHPKSgLxGo9YuQzz20o+w==", + "dev": true + }, "keygrip": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/keygrip/-/keygrip-1.1.0.tgz", @@ -7324,6 +7372,7 @@ "resolved": "https://registry.npmjs.org/keytar/-/keytar-7.9.0.tgz", "integrity": "sha512-VPD8mtVtm5JNtA2AErl6Chp06JBfy7diFQ7TQQhdpWOl6MrCRB+eRbvAZUsbGQS9kiMq0coJsy0W0vHpDCkWsQ==", "dev": true, + "optional": true, "requires": { "node-addon-api": "^4.3.0", "prebuild-install": "^7.0.1" @@ -7651,7 +7700,8 @@ "version": "3.1.0", "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==", - "dev": true + "dev": true, + "optional": true }, "minimatch": { "version": "3.1.2", @@ -7672,7 +7722,8 @@ "version": "0.5.3", "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==", - "dev": true + "dev": true, + "optional": true }, "mocha": { "version": "10.1.0", @@ -7826,7 +7877,8 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-1.0.2.tgz", "integrity": "sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg==", - "dev": true + "dev": true, + "optional": true }, "native-ext-loader": { "version": "2.3.0", @@ -7857,6 +7909,7 @@ "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.28.0.tgz", "integrity": "sha512-fRlDb4I0eLcQeUvGq7IY3xHrSb0c9ummdvDSYWfT9+LKP+3jCKw/tKoqaM7r1BAoiAC6GtwyjaGnOz6B3OtF+A==", "dev": true, + "optional": true, "requires": { "semver": "^7.3.5" }, @@ -7866,6 +7919,7 @@ "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz", "integrity": "sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==", "dev": true, + "optional": true, "requires": { "lru-cache": "^6.0.0" } @@ -7876,7 +7930,8 @@ "version": "4.3.0", "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-4.3.0.tgz", "integrity": "sha512-73sE9+3UaLYYFmDsFZnqCInzPyh3MqIwZO9cw58yIqAZhONrrabrYyYe3TuIqtIiOuTXVhsGau8hcrhhwSsDIQ==", - "dev": true + "dev": true, + "optional": true }, "node-domexception": { "version": "1.0.0", @@ -8182,6 +8237,7 @@ "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.1.tgz", "integrity": "sha512-jAXscXWMcCK8GgCoHOfIr0ODh5ai8mj63L2nWrjuAgXE6tDyYGnx4/8o/rCgU+B4JSyZBKbeZqzhtwtC3ovxjw==", "dev": true, + "optional": true, "requires": { "detect-libc": "^2.0.0", "expand-template": "^2.0.3", @@ -8220,6 +8276,7 @@ "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", "dev": true, + "optional": true, "requires": { "end-of-stream": "^1.1.0", "once": "^1.3.1" @@ -8260,6 +8317,7 @@ "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", "dev": true, + "optional": true, "requires": { "deep-extend": "^0.6.0", "ini": "~1.3.0", @@ -8271,7 +8329,8 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", - "dev": true + "dev": true, + "optional": true } } }, @@ -8551,13 +8610,15 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz", "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==", - "dev": true + "dev": true, + "optional": true }, "simple-get": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.1.tgz", "integrity": "sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==", "dev": true, + "optional": true, "requires": { "decompress-response": "^6.0.0", "once": "^1.3.1", @@ -8664,6 +8725,7 @@ "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.1.tgz", "integrity": "sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==", "dev": true, + "optional": true, "requires": { "chownr": "^1.1.1", "mkdirp-classic": "^0.5.2", @@ -8676,6 +8738,7 @@ "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", "dev": true, + "optional": true, "requires": { "buffer": "^5.5.0", "inherits": "^2.0.4", @@ -8687,6 +8750,7 @@ "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", "dev": true, + "optional": true, "requires": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", @@ -8698,6 +8762,7 @@ "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", "dev": true, + "optional": true, "requires": { "bl": "^4.0.3", "end-of-stream": "^1.4.1", @@ -8830,6 +8895,7 @@ "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", "integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==", "dev": true, + "optional": true, "requires": { "safe-buffer": "^5.0.1" } @@ -8958,9 +9024,9 @@ "dev": true }, "webpack": { - "version": "5.75.0", - "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.75.0.tgz", - "integrity": "sha512-piaIaoVJlqMsPtX/+3KTTO6jfvrSYgauFVdt8cr9LTHKmcq/AMd4mhzsiP7ZF/PGRNPGA8336jldh9l2Kt2ogQ==", + "version": "5.76.1", + "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.76.1.tgz", + "integrity": "sha512-4+YIK4Abzv8172/SGqObnUjaIHjLEuUasz9EwQj/9xmPPkYJy2Mh03Q/lJfSD3YLzbxy5FeTq5Uw0323Oh6SJQ==", "dev": true, "requires": { "@types/eslint-scope": "^3.7.3", diff --git a/package.json b/package.json index 69a8a42..440d512 100644 --- a/package.json +++ b/package.json @@ -1,8 +1,8 @@ { "name": "databricks-vscode", - "displayName": "Databricks VSCode", + "displayName": "Databricks Power Tools", "description": "Run notebooks cell-by-cell, browse and edit your Databricks Workspace, DBFS, Clusters, Jobs, Secrets, Repos and SQL. Supports Azure Databricks, Databricks on AWS and Databricks on GCP.", - "version": "1.5.1", + "version": "2.0.0", "publisher": "paiqo", "icon": "resources/databricks_extension.png", "author": { @@ -34,6 +34,7 @@ "Azure", "Notebooks", "Data Science", + "Machine Learning", "Other" ], "keywords": [ @@ -51,12 +52,13 @@ ], "activationEvents": [ "onFileSystem:dbfs", - "onFileSystem:dbws" + "onFileSystem:dbws", + "onFileSystem:wsfs" ], "capabilities": { "virtualWorkspaces": { "supported": "limited", - "description": "In virtual workspaces, Workspace and DBFS browsers are not supported. Please use `dbfs:/` and `dbws:/` mount points instead." + "description": "In virtual workspaces, Workspace and DBFS browsers are not supported. Please use `dbfs:/` and `wsfs:/` mount points instead." }, "untrustedWorkspaces": { "supported": "limited", @@ -68,23 +70,27 @@ "contributes": { "configuration": [ { - "name": "Databricks", + "name": "paiqo.databricks-vscode", "type": "object", - "title": "Databricks", + "title": "Databricks Power Tools", "properties": { "databricks.connectionManager": { "type": "string", "enum": [ + "Default", "VSCode Settings", "Databricks CLI Profiles", "Azure", - "ManualInput" + "Databricks Extension", + "Manual" ], - "default": "VSCode Settings", + "default": "Default", "enumDescriptions": [ + "The best matching connection manager is selected automatically.", "All connection information is stored securely in the VSCode Settings (user or workspace).", "Connection information is read from the Databricks CLI config file. Databricks CLI has to be configured upfront! Also supports the environment varialbe 'DATABRICKS_CONFIG_FILE'.", "Connection information is loaded from Azure. All Azure Databricks workspaces that you have access to will show up atuomatically in the connection list.", + "A single connection that is managed by the Databricks Extensions", "You are prompted for connection information once you interact with the extension." ], "description": "Defines how connection information is managed and stored.", @@ -94,16 +100,14 @@ "type": "string", "enum": [ "VSCodeSettings", - "SystemKeyChain", - "ExternalConfigFile" + "SystemKeyChain" ], "default": "SystemKeyChain", "enumDescriptions": [ "Databricks Personal Access Token is stored in the VSCode Settings as plain text! There is a potential risk that the PAT gets checked in to source control systems like git!", - "Databricks Personal Access Token is securely stored in the System Key Chain. Easy Copying/Sharing connections with other users is not possible this way. VSCode only stores a reference to the the Systems KeyChain entry.", - "(Experimental!) Databricks Personal Access Token is stored in an external Config file. This is useful to manage all your PATs in one place and also easily share them." + "Databricks Personal Access Token is securely stored in the System Key Chain. Easy Copying/Sharing connections with other users is not possible this way. VSCode only stores a reference to the the Systems KeyChain entry." ], - "description": "Defines the default how the VSCode extension for Databricks stores sensitive values like the Databricks Personal Access Token (PAT). Only releveant if the VSCode Connection Manager is used!", + "markdownDescription": "(`VSCode Settings`) Defines the default how the VSCode extension for Databricks stores sensitive values like the Databricks Personal Access Token (PAT). Only releveant if the VSCode Connection Manager is used!", "scope": "window" }, "databricks.connection.default.exportFormats": { @@ -114,7 +118,7 @@ "SQL": ".sql", "R": ".r" }, - "description": "(Optional) The different file extension used when downloading notebooks from Databricks.", + "markdownDescription": "(`VSCode Settings`, Optional) The different file extension used when downloading notebooks from Databricks.", "scope": "window", "properties": { "Scala": { @@ -151,7 +155,7 @@ }, "databricks.connection.default.displayName": { "type": "string", - "description": "(Mandatory) Name shown in the Connection list of the Databricks VSCode extension. Must be unique across all VSCode workspaces to avoid conflicts!", + "markdownDescription": "(`VSCode Settings`, Mandatory) Name shown in the Connection list of the Databricks VSCode extension. Must be unique across all VSCode workspaces to avoid conflicts!", "scope": "window", "examples": [ "my Connection", @@ -161,7 +165,7 @@ }, "databricks.connection.default.apiRootUrl": { "type": "string", - "description": "(Mandatory) Root URL of the Databricks API. Azure: https://adb-12345678901234.19.azuredatabricks.net - AWS: https://abc-12345-xaz.cloud.databricks.com. Please do not provide any path or querystring like `?o=123456`", + "markdownDescription": "(`VSCode Settings`, Mandatory) Root URL of the Databricks API. Azure: https://adb-12345678901234.19.azuredatabricks.net - AWS: https://abc-12345-xaz.cloud.databricks.com. Please do not provide any path or querystring like `?o=123456`", "scope": "window", "examples": [ "https://adb-12345678901234.19.azuredatabricks.net/", @@ -170,7 +174,7 @@ }, "databricks.connection.default.personalAccessToken": { "type": "string", - "description": "(Mandatory) The Personal Access Token (PAT) used to access the Databricks API.", + "markdownDescription": "(`VSCode Settings`, Mandatory) The Personal Access Token (PAT) used to access the Databricks API.", "scope": "window", "examples": [ "dapi0123456789abcdef0123456789abcdef" @@ -178,7 +182,7 @@ }, "databricks.connection.default.personalAccessTokenSecure": { "type": "object", - "description": "(Optional) Reference to a secure store for the Personal Access Token.", + "markdownDescription": "(`VSCode Settings`, Optional) Reference to a secure store for the Personal Access Token.", "scope": "window", "maxProperties": 1, "properties": { @@ -196,7 +200,7 @@ }, "databricks.connection.default.localSyncFolder": { "type": "string", - "description": "(Mandatory) A local path where files (notebooks) that are downloaded from Databricks should be stored. This folder can then be integrated in your GIT repository. \nExamples: \nWindows - C:\\mySyncFolder \nLinux/Unix - /home/myUser/mySyncFolder \nmacOS - myUser/mySyncFolder (Please do not use leading / on macOS!)", + "markdownDescription": "(`VSCode Settings`, Mandatory) A local path where files (notebooks) that are downloaded from Databricks should be stored. This folder can then be integrated in your GIT repository. \nExamples: \nWindows - C:\\mySyncFolder \nLinux/Unix - /home/myUser/mySyncFolder \nmacOS - myUser/mySyncFolder (Please do not use leading / on macOS!)", "scope": "window", "examples": [ "Windows: C:\\mySyncFolder", @@ -212,7 +216,7 @@ "DBFS": "DBFS", "Jobs": "Jobs" }, - "description": "(Optional) Can be used to specify different sub-paths for the items that can be downloaded from Databricks.", + "markdownDescription": "(`VSCode Settings`, Optional) Can be used to specify different sub-paths for the items that can be downloaded from Databricks.", "scope": "window", "properties": { "Workspace": { @@ -235,13 +239,13 @@ }, "databricks.connection.default.useCodeCells": { "type": "boolean", - "description": "If true, Code Cell tags (# %%) will be added when downloading raw source files.", + "markdownDescription": "(`VSCode Settings`, Optional) If true, Code Cell tags (# %%) will be added when downloading raw source files.", "scope": "window", "default": true }, "databricks.connections": { "type": "array", - "description": "An array of objects where each object represents a Databricks Connection containing the same settings as defined for databricks.connection.default.* ", + "makrdownDescription": "(`VSCode Settings`) An array of objects where each object represents a Databricks Connection containing the same settings as defined for databricks.connection.default.* ", "items": { "type": "object", "properties": { @@ -364,17 +368,17 @@ }, "databricks.lastActiveConnection": { "type": "string", - "description": "(Optional) The displayName of the Connection that was used in the last session.", + "description": "The displayName of the Connection that was used in the last session.", "scope": "window" }, "databricks.azure.tenantId": { "type": "string", - "description": "(Optional) To connect to Azure Databricks Workspaces outside of your home-tenant.", + "markdownDescription": "(`Azure`, Optional) To connect to Azure Databricks Workspaces outside of your home-tenant.", "scope": "window" }, "databricks.azure.subscriptionIds": { "type": "array", - "description": "(Optional) A list of Azure Subscription IDs (GUIDs) from which the Databricks Workspaces are loaded. If not specified, all accessible subscriptions are searched.", + "markdownDescription": "(`Azure`, Optional) A list of Azure Subscription IDs (GUIDs) from which the Databricks Workspaces are loaded. If not specified, all accessible subscriptions are searched.", "scope": "window", "items": { "type": "string", @@ -386,7 +390,7 @@ }, "databricks.azure.workspaces": { "type": "array", - "description": "(Optional) A list of Azure Resource IDs to load directly without browsing the tenant first.", + "markdownDescription": "(`Azure`, Optional) A list of Azure Resource IDs to load directly without browsing the tenant first.", "scope": "window", "items": { "type": "object", @@ -415,25 +419,26 @@ "viewsContainers": { "activitybar": [ { - "id": "databricks", - "title": "Databricks", + "id": "databricksPowerTools", + "title": "Databricks Power Tools", "icon": "resources/databricks_sidebar.png" } ] }, "views": { - "databricks": [ + "databricksPowerTools": [ { "id": "databricksConnections", "name": "Connections", "type": "tree", - "initialSize": 25 + "initialSize": 25, + "when": "!paiqo.databricks.hideConnectionManager" }, { "id": "databricksWorkspace", "name": "Workspace", "type": "tree", - "when": "!virtualWorkspace" + "when": "!virtualWorkspace && !paiqo.databricks.isInBrowser" }, { "id": "databricksClusters", @@ -449,7 +454,7 @@ "id": "databricksFS", "name": "DBFS", "type": "tree", - "when": "!virtualWorkspace" + "when": "!virtualWorkspace && !paiqo.databricks.isInBrowser" }, { "id": "databricksSecrets", @@ -468,27 +473,37 @@ } ] }, + "viewsWelcome": [ + { + "view": "databricksSQL", + "contents": "Please select a cluster first" + } + ], "commands": [ { "command": "databricksConnections.refresh", "title": "Refresh", + "category": "Databricks Power Tools", "icon": "$(refresh)" }, { "command": "databricksConnections.add", "title": "Add", + "category": "Databricks Power Tools", "icon": "$(add)", "enablement": "virtualWorkspace" }, { "command": "databricksConnections.settings", "title": "Settings", + "category": "Databricks Power Tools", "icon": "$(gear)", "enablement": "!virtualWorkspace" }, { "command": "databricksConnectionItem.activate", "title": "Activate", + "category": "Databricks Power Tools", "icon": { "light": "resources/light/connected.png", "dark": "resources/dark/connected.png" @@ -497,72 +512,87 @@ { "command": "databricksWorkspace.refresh", "title": "Refresh", + "category": "Databricks Power Tools", "icon": "$(refresh)" }, { "command": "databricksWorkspace.download", "title": "Download", + "category": "Databricks Power Tools", "icon": "$(cloud-download)" }, { "command": "databricksWorkspace.upload", "title": "Upload", + "category": "Databricks Power Tools", "icon": "$(cloud-upload)" }, { "command": "databricksWorkspace.addToWorkspace", "title": "Add Databricks Workspace to VSCode workspace", + "category": "Databricks Power Tools", "icon": "$(symbol-class)" }, { "command": "databricksWorkspaceItem.click", - "title": "Open File" + "title": "Open File", + "category": "Databricks Power Tools" }, { "command": "databricksWorkspaceItem.download", "title": "Download", + "category": "Databricks Power Tools", "icon": "$(cloud-download)" }, { "command": "databricksWorkspaceItem.upload", "title": "Upload", + "category": "Databricks Power Tools", "icon": "$(cloud-upload)" }, { "command": "databricksWorkspaceItem.compare", "title": "Compare", + "category": "Databricks Power Tools", "icon": "$(compare-changes)" }, { "command": "databricksWorkspaceItem.delete", "title": "Delete", + "category": "Databricks Power Tools", "icon": "$(remove-close)" }, { "command": "databricksWorkspaceItem.copyPath", - "title": "Copy Path" + "title": "Copy Path", + "category": "Databricks Power Tools" }, { "command": "databricksWorkspaceItem.openExplorer", - "title": "Open Explorer" + "title": "Open Explorer", + "category": "Databricks Power Tools" }, { "command": "databricksClusters.refresh", "title": "Refresh", + "category": "Databricks Power Tools", "icon": "$(refresh)" }, { "command": "databricksClusters.add", "title": "Add", + "category": "Databricks Power Tools", "icon": "$(add)" }, { "command": "databricksClusterItem.click", - "title": "Open File" + "title": "Open File", + "category": "Databricks Power Tools" }, { "command": "databricksClusterItem.start", "title": "Start", + "category": "Databricks Power Tools", "icon": { "light": "resources/light/start.png", "dark": "resources/dark/start.png" @@ -571,6 +601,7 @@ { "command": "databricksClusterItem.stop", "title": "Stop", + "category": "Databricks Power Tools", "icon": { "light": "resources/light/stop.png", "dark": "resources/dark/stop.png" @@ -579,58 +610,77 @@ { "command": "databricksClusterItem.showDefinition", "title": "JSON Definition", + "category": "Databricks Power Tools", "icon": "$(json)" }, { "command": "databricksClusterItem.delete", "title": "Delete", + "category": "Databricks Power Tools", "icon": "$(remove-close)" }, { "command": "databricksClusterItem.pin", "title": "Pin", + "category": "Databricks Power Tools", "icon": "$(pin)" }, { "command": "databricksClusterItem.unpin", "title": "Unpin", + "category": "Databricks Power Tools", "icon": "$(pinned)" }, { "command": "databricksClusterItem.useForSQL", "title": "Use for SQL Browser", + "category": "Databricks Power Tools", "icon": { "light": "resources/light/stop.png", "dark": "resources/dark/stop.png" - } + }, + "enablement": "paiqo.databricks.connectionManager != 'Databricks Extension'" + }, + { + "command": "databricksClusterItem.attachCluster", + "title": "Attach cluster", + "category": "Databricks Power Tools", + "icon": "$(plug)", + "enablement": "paiqo.databricks.connectionManager == 'Databricks Extension'" }, { "command": "databricksClusterItem.createKernel", "title": "Create Databricks Cluster Kernel", - "shortTitle": "Create Databricks Kernel" + "shortTitle": "Create Databricks Kernel", + "category": "Databricks Power Tools" }, { "command": "databricksClusterItem.restartKernel", "title": "Restart Databricks Cluster Kernel", - "shortTitle": "Restart Databricks Kernel" + "shortTitle": "Restart Databricks Kernel", + "category": "Databricks Power Tools" }, { "command": "databricksJobs.refresh", "title": "Refresh", + "category": "Databricks Power Tools", "icon": "$(refresh)" }, { "command": "databricksJobItem.showDefinition", "title": "JSON Definition", + "category": "Databricks Power Tools", "icon": "$(json)" }, { "command": "databricksJobItem.click", - "title": "Open File" + "title": "Open File", + "category": "Databricks Power Tools" }, { "command": "databricksJobItem.start", "title": "Start", + "category": "Databricks Power Tools", "icon": { "light": "resources/light/start.png", "dark": "resources/dark/start.png" @@ -639,6 +689,7 @@ { "command": "databricksJobItem.stop", "title": "Stop", + "category": "Databricks Power Tools", "icon": { "light": "resources/light/stop.png", "dark": "resources/dark/stop.png" @@ -647,124 +698,149 @@ { "command": "databricksJobItem.openBrowser", "title": "Open Browser", + "category": "Databricks Power Tools", "icon": "$(globe)" }, { "command": "databricksFS.addToWorkspace", "title": "Add DBFS to VSCode workspace", + "category": "Databricks Power Tools", "icon": "$(symbol-class)" }, { "command": "databricksFS.refresh", "title": "Refresh", + "category": "Databricks Power Tools", "icon": "$(refresh)" }, { "command": "databricksFS.addFile", "title": "Add", + "category": "Databricks Power Tools", "icon": "$(new-file)" }, { "command": "databricksFS.addDirectory", "title": "Add", + "category": "Databricks Power Tools", "icon": "$(file-directory-create)" }, { "command": "databricksFSItem.download", "title": "Download", + "category": "Databricks Power Tools", "icon": "$(cloud-download)" }, { "command": "databricksFSItem.upload", "title": "Upload", + "category": "Databricks Power Tools", "icon": "$(cloud-upload)" }, { "command": "databricksFSItem.click", - "title": "Preview File" + "title": "Preview File", + "category": "Databricks Power Tools" }, { "command": "databricksFSItem.addFile", "title": "Add", + "category": "Databricks Power Tools", "icon": "$(new-file)" }, { "command": "databricksFSItem.addDirectory", "title": "Add Directory", + "category": "Databricks Power Tools", "icon": "$(file-directory-create)" }, { "command": "databricksFSItem.delete", "title": "Delete", + "category": "Databricks Power Tools", "icon": "$(remove-close)" }, { "command": "databricksFSItem.copyPath", - "title": "Copy Path" + "title": "Copy Path", + "category": "Databricks Power Tools" }, { "command": "databricksSecrets.refresh", "title": "Refresh", + "category": "Databricks Power Tools", "icon": "$(refresh)" }, { "command": "databricksSecrets.addSecretScope", "title": "Add Secret Scope", + "category": "Databricks Power Tools", "icon": "$(add)" }, { "command": "databricksSecretScope.delete", "title": "Delete", + "category": "Databricks Power Tools", "icon": "$(remove-close)" }, { "command": "databricksSecretScope.addSecret", "title": "Add Secret", + "category": "Databricks Power Tools", "icon": "$(add)" }, { "command": "databricksSecret.update", "title": "Update", + "category": "Databricks Power Tools", "icon": "$(pencil)" }, { "command": "databricksSecret.delete", "title": "Delete", + "category": "Databricks Power Tools", "icon": "$(remove-close)" }, { "command": "databricksSecret.insertCode", "title": "Insert", + "category": "Databricks Power Tools", "icon": "$(code)" }, { "command": "databricksSQL.refresh", "title": "Refresh", + "category": "Databricks Power Tools", "icon": "$(refresh)" }, { "command": "databricksSQLTable.showDefinition", "title": "Show Definition", + "category": "Databricks Power Tools", "icon": "$(json)" }, { "command": "databricksRepos.refresh", "title": "Refresh", + "category": "Databricks Power Tools", "icon": "$(refresh)" }, { "command": "databricksRepo.pull", "title": "Pull", + "category": "Databricks Power Tools", "icon": "$(repo-pull)" }, { "command": "databricksRepo.checkOut", "title": "Check-Out", + "category": "Databricks Power Tools", "icon": "$(repo-clone)" }, { "command": "databricksRepo.delete", "title": "Delete", + "category": "Databricks Power Tools", "icon": "$(remove-close)" }, { @@ -942,7 +1018,12 @@ }, { "command": "databricksClusterItem.useForSQL", - "when": "view == databricksClusters && viewItem =~ /.*,STARTED,.*/ && viewItem =~ /.*,MANUAL,.*/" + "when": "view == databricksClusters && viewItem =~ /.*,STARTED,.*/ && viewItem =~ /.*,MANUAL,.*/ && paiqo.databricks.connectionManager != 'Databricks Extension'" + }, + { + "command": "databricksClusterItem.attachCluster", + "when": "view == databricksClusters && viewItem =~ /.*,MANUAL,.*/ && paiqo.databricks.connectionManager == 'Databricks Extension'", + "group": "inline" }, { "command": "databricksClusterItem.createKernel", @@ -1094,7 +1175,7 @@ "@types/node": "^16.11.7", "@types/rimraf": "^2.0.5", "@types/vscode": "^1.68.0", - "@vscode/vsce": "^2.15.0", + "@vscode/vsce": "^2.18.0", "@vscode/test-web": "*", "eslint": "^8.19.0", "glob": "^8.0.3", @@ -1105,7 +1186,7 @@ "process": "^0.11.10", "ts-loader": "^9.3.1", "typescript": "^4.7.4", - "webpack": "^5.74.0", + "webpack": "^5.76.0", "webpack-cli": "^4.10.0" } } \ No newline at end of file diff --git a/src/ThisExtension.ts b/src/ThisExtension.ts index bc213dd..a5f025e 100644 --- a/src/ThisExtension.ts +++ b/src/ThisExtension.ts @@ -4,19 +4,26 @@ import { WorkspaceItemLanguage } from './vscode/treeviews/workspaces/_types'; import { DatabricksConnectionManager } from './vscode/treeviews/connections/DatabricksConnectionManager'; import { iDatabricksConnection } from './vscode/treeviews/connections/iDatabricksConnection'; import { DatabricksConnectionManagerVSCode } from './vscode/treeviews/connections/DatabricksConnectionManagerVSCode'; -import { SensitiveValueStore } from './vscode/treeviews/connections/_types'; +import { ConnectionManager, ConnectionSource, SensitiveValueStore } from './vscode/treeviews/connections/_types'; import { DatabricksConnectionManagerCLI } from './vscode/treeviews/connections/DatabricksConnectionManagerCLI'; import { DatabricksConnectionManagerManualInput } from './vscode/treeviews/connections/DatabricksConnectionManagerManualInput'; import { DatabricksConnectionManagerAzure } from './vscode/treeviews/connections/DatabricksConnectionManagerAzure'; +import { DatabricksConnectionManagerDatabricks } from './vscode/treeviews/connections/DatabricksConnectionManagerDatabricks'; // https://vshaxe.github.io/vscode-extern/vscode/TreeDataProvider.html export abstract class ThisExtension { + static readonly DBFS_SCHEME = "dbfs"; + static readonly WORKSPACE_SCHEME = "wsfs"; + static readonly WORKSPACE_SCHEME_LEGACY = "dbws"; + + private static _context: vscode.ExtensionContext; private static _extension: vscode.Extension; private static _statusBar: vscode.StatusBarItem; private static _isValidated: boolean = false; private static _logger: vscode.OutputChannel; + private static _connectionManagerText: ConnectionManager; private static _connectionManager: DatabricksConnectionManager; private static _settingScope: ConfigSettingSource; private static _sensitiveValueStore: SensitiveValueStore; @@ -61,8 +68,29 @@ export abstract class ThisExtension { ThisExtension.readGlobalSettings(); - let connectionManager = this.getConfigurationSetting("databricks.connectionManager", this.SettingScope, true); - switch (connectionManager.value) { + this._connectionManagerText = undefined; + let connectionManager = this.getConfigurationSetting("databricks.connectionManager", this.SettingScope, true); + let conManager: ConnectionManager = connectionManager.value; + + // handle default + if (conManager == "Default") { + ThisExtension.log("Default connection manager selected. Trying to find the best connection manager ...") + if (vscode.extensions.getExtension("databricks.databricks")) { + ThisExtension.log("Databricks Extension found. Using it as connection manager ..."); + conManager = "Databricks Extension"; + } + else { + ThisExtension.log("Databricks Extension not found. Using VSCode Settings as connection manager ..."); + conManager = "VSCode Settings"; + // should open workspace settings with a filter but the filter is not yet working + vscode.commands.executeCommand("workbench.action.openWorkspaceSettings", ThisExtension.configuration.id); + } + + this._connectionManagerText = conManager; + } + + switch (conManager) { + case "VSCode Settings": this._connectionManager = new DatabricksConnectionManagerVSCode(); break; @@ -72,16 +100,23 @@ export abstract class ThisExtension { case "Azure": this._connectionManager = new DatabricksConnectionManagerAzure(); break; - case "ManualInput": + case "Databricks Extension": + this._connectionManager = new DatabricksConnectionManagerDatabricks(); + break; + case "Manual": this._connectionManager = new DatabricksConnectionManagerManualInput(); break; default: this.log("'" + connectionManager + "' is not a valid value for config setting 'databricks.connectionManager!"); - + } + this._connectionManagerText = conManager + await this.ConnectionManager.initialize(); + await this.setContext(); + return true; } catch (error) { return false; @@ -92,6 +127,27 @@ export abstract class ThisExtension { } + private static async setContext(): Promise { + // we hide the Connections Tab as we load all information from the Databricks Extension + await vscode.commands.executeCommand( + "setContext", + "paiqo.databricks.hideConnectionManager", + this._connectionManagerText == "Databricks Extension" + ); + + await vscode.commands.executeCommand( + "setContext", + "paiqo.databricks.connectionManager", + this._connectionManagerText + ); + + await vscode.commands.executeCommand( + "setContext", + "paiqo.databricks.isInBrowser", + this.isInBrowser + ); + } + private static readGlobalSettings(): void { let settingScope: ConfigSettingSource = "Workspace"; @@ -126,6 +182,10 @@ export abstract class ThisExtension { return this._connectionManager; } + static get ConnectionManagerText(): ConnectionManager { + return this._connectionManagerText; + } + static get SQLClusterID(): string { return this._sqlClusterId; } @@ -251,7 +311,8 @@ export abstract class ThisExtension { } static get isInBrowser(): boolean { - return process.hasOwnProperty("browser") && process["browser"]; + return vscode.env.uiKind === vscode.UIKind.Web; + //return process.hasOwnProperty("browser") && process["browser"]; } static async updateConfigurationSetting(setting: string, value: any, target: ConfigSettingSource = this._settingScope): Promise { diff --git a/src/databricksApi/databricksApiService.ts b/src/databricksApi/databricksApiService.ts index fab248f..f3ba897 100644 --- a/src/databricksApi/databricksApiService.ts +++ b/src/databricksApi/databricksApiService.ts @@ -40,11 +40,7 @@ export abstract class DatabricksApiService { let headers = await ThisExtension.ConnectionManager.getAuthorizationHeaders(con); this._apiBaseUrl = Helper.trimChar(con.apiRootUrl.with({ path: '', query: '', fragment: '' }).toString(true), '/') + this.API_SUB_URL; - this._headers = { - ...headers, - "Content-Type": 'application/json', - "Accept": 'application/json' - } + this.updateHeaders(headers); ThisExtension.log(`Testing new Databricks API (${con.apiRootUrl}) settings ...`); this._connectionTestRunning = true; @@ -71,6 +67,14 @@ export abstract class DatabricksApiService { public static get isInitialized(): boolean { return DatabricksApiService._isInitialized; } + + public static updateHeaders(authorizationHeaders: any): void { + this._headers = { + ...authorizationHeaders, + "Content-Type": 'application/json', + "Accept": 'application/json' + } + } //#endregion //#region Helpers diff --git a/src/extension.ts b/src/extension.ts index eb97ea5..f522e41 100644 --- a/src/extension.ts +++ b/src/extension.ts @@ -69,7 +69,6 @@ export async function activate(context: vscode.ExtensionContext) { ); ThisExtension.setStatusBar("Kernels initialized!"); - // register DatabricksConnectionTreeProvider let databricksConnectionTreeProvider = new DatabricksConnectionTreeProvider(); vscode.window.registerTreeDataProvider('databricksConnections', databricksConnectionTreeProvider); @@ -82,7 +81,6 @@ export async function activate(context: vscode.ExtensionContext) { } vscode.commands.registerCommand('databricksConnectionItem.activate', (connection: DatabricksConnectionTreeItem) => connection.activate()); - // register DatabricksWorkspaceTreeProvider if (!ThisExtension.isInBrowser) { let databricksWorkspaceTreeProvider = new DatabricksWorkspaceTreeProvider(context); @@ -102,11 +100,11 @@ export async function activate(context: vscode.ExtensionContext) { // Workspace File System Provider const workspaceProvider = new DatabricksWorkspaceProvider(context); vscode.commands.registerCommand('databricksWorkspace.addToWorkspace', (showMessage: boolean) => { - FSHelper.addToWorkspace(vscode.Uri.parse('dbws:/'), "Databricks - Workspace", showMessage); + FSHelper.addToWorkspace(vscode.Uri.parse(ThisExtension.WORKSPACE_SCHEME +':/'), "Databricks - Workspace", showMessage); }); if (ThisExtension.isInBrowser) { - // in a virtual workspace we always want to add the DBWS mount to the workspace + // in a virtual workspace we always want to add the WSFS/DBWS mount to the workspace vscode.commands.executeCommand('databricksWorkspace.addToWorkspace', false); } @@ -127,6 +125,9 @@ export async function activate(context: vscode.ExtensionContext) { vscode.commands.registerCommand('databricksClusterItem.createKernel', (cluster: DatabricksCluster) => cluster.createKernel()); vscode.commands.registerCommand('databricksClusterItem.restartKernel', (cluster: DatabricksCluster) => cluster.restartKernel()); + // set cluster for Databricks Extension + vscode.commands.registerCommand('databricksClusterItem.attachCluster', (cluster: DatabricksCluster) => cluster.attachCluster()); + // register DatabricksJobsTreeProvider let databricksJobsTreeProvider = new DatabricksJobTreeProvider(context); @@ -159,11 +160,11 @@ export async function activate(context: vscode.ExtensionContext) { // DBFS File System Provider const dbfsProvider = new DatabricksFileSystemProvider(context); vscode.commands.registerCommand('databricksFS.addToWorkspace', (showMessage: boolean) => { - FSHelper.addToWorkspace(vscode.Uri.parse('dbfs:/'), "Databricks - DBFS", showMessage); + FSHelper.addToWorkspace(vscode.Uri.parse(ThisExtension.DBFS_SCHEME + ':/'), "Databricks - DBFS", showMessage); }); if (ThisExtension.isInBrowser) { - // in a virtual workspace we always want to add the DBWS mount to the workspace + // in a virtual workspace we always want to add the DBFS mount to the workspace vscode.commands.executeCommand('databricksFS.addToWorkspace', false); } diff --git a/src/helpers/Helper.ts b/src/helpers/Helper.ts index 2415a7a..8aff3e6 100644 --- a/src/helpers/Helper.ts +++ b/src/helpers/Helper.ts @@ -73,8 +73,7 @@ export abstract class Helper { try { // three '/' in the beginning indicate a local path // however, there are issues if this.localFilePath also starts with a '/' so we do a replace in this special case - if(typeof(path) == "string") - { + if (typeof (path) == "string") { path = vscode.Uri.parse(("file:///" + path).replace('////', '///')); } await vscode.workspace.fs.stat(path); @@ -100,8 +99,7 @@ export abstract class Helper { static getToken(text: string, separator: string, token: number): string { let parts: string[] = text.split(separator); - if(token < 0) - { + if (token < 0) { return parts.slice(token)[0]; } return parts[token]; @@ -110,8 +108,7 @@ export abstract class Helper { // not working! static runAsyncFunction(func: Function, args: any = undefined): T { return (async function () { - if(args == undefined) - { + if (args == undefined) { return func() } return func(args); @@ -127,7 +124,7 @@ export abstract class Helper { } return immediatelyResolvedPromise as unknown as T; } - + static ensureLocalFolder(path: string, pathIsFile: boolean = false): void { let folder: vscode.Uri = vscode.Uri.file(path); @@ -196,7 +193,7 @@ export abstract class Helper { vscode.commands.executeCommand("vscode.diff", filePath1, filePath2, "Online <-> Local", options); } - + static openLink(link: string): void { vscode.commands.executeCommand('vscode.open', vscode.Uri.parse(link)); @@ -261,13 +258,12 @@ export abstract class Helper { var mDisplay = m > 0 ? `${m.toString().length > 1 ? `${m}` : `0${m}`}` : '00'; var sDisplay = s > 0 ? `${s.toString().length > 1 ? `${s}` : `0${s}`}` : '00'; - return `${hDisplay}:${mDisplay}:${sDisplay}`; + return `${hDisplay}:${mDisplay}:${sDisplay}`; } static getFirstRegexGroup(regexp: RegExp, text: string): string { const array = [...text.matchAll(regexp)]; - if(array.length >= 1) - { + if (array.length >= 1) { return array[0][1]; } return null; @@ -276,4 +272,48 @@ export abstract class Helper { static parseBoolean(value: string): boolean { return value === 'false' || value === 'undefined' || value === 'null' || value === '0' ? false : !!value; } + + static async waitForPromise(promise: Promise, timeout: number): Promise { + // Set a timer that will resolve with null + return new Promise((resolve, reject) => { + const timer = setTimeout(() => resolve(null), timeout); + promise + .then((result) => { + // When the promise resolves, make sure to clear the timer or + // the timer may stick around causing tests to wait + clearTimeout(timer); + resolve(result); + }) + .catch((e) => { + clearTimeout(timer); + reject(e); + }); + }); + } + + static async waitForCondition( + condition: () => Promise, + timeout: number, + interval: number + ): Promise { + // Set a timer that will resolve with null + return new Promise((resolve) => { + let finish: (result: boolean) => void; + const timer = setTimeout(() => finish(false), timeout); + const intervalId = setInterval(() => { + condition() + .then((r) => { + if (r) { + finish(true); + } + }) + .catch((_e) => finish(false)); + }, interval); + finish = (result: boolean) => { + clearTimeout(timer); + clearInterval(intervalId); + resolve(result); + }; + }); + } } \ No newline at end of file diff --git a/src/vscode/filesystemProvider/DatabricksFileSystemProvider.ts b/src/vscode/filesystemProvider/DatabricksFileSystemProvider.ts index 55d5532..f783db1 100644 --- a/src/vscode/filesystemProvider/DatabricksFileSystemProvider.ts +++ b/src/vscode/filesystemProvider/DatabricksFileSystemProvider.ts @@ -8,12 +8,13 @@ import { DatabricksApiService } from '../../databricksApi/databricksApiService'; import { iDatabricksApiDbfsReadResponse } from '../../databricksApi/_types'; import { FSHelper } from '../../helpers/FSHelper'; import { Helper } from '../../helpers/Helper'; +import { ThisExtension } from '../../ThisExtension'; import { iDatabricksFSItem } from '../treeviews/dbfs/iDatabricksFSItem'; export class DatabricksFileSystemProvider implements vscode.FileSystemProvider { constructor(context: vscode.ExtensionContext) { - context.subscriptions.push(vscode.workspace.registerFileSystemProvider('dbfs', this, { isCaseSensitive: true })); + context.subscriptions.push(vscode.workspace.registerFileSystemProvider(ThisExtension.DBFS_SCHEME, this, { isCaseSensitive: true })); } // --- manage file metadata diff --git a/src/vscode/filesystemProvider/DatabricksWorkspaceProvider.ts b/src/vscode/filesystemProvider/DatabricksWorkspaceProvider.ts index 446a085..172f535 100644 --- a/src/vscode/filesystemProvider/DatabricksWorkspaceProvider.ts +++ b/src/vscode/filesystemProvider/DatabricksWorkspaceProvider.ts @@ -7,6 +7,7 @@ import { DatabricksApiService } from '../../databricksApi/databricksApiService'; import { iDatabricksWorkspaceItem } from '../treeviews/workspaces/iDatabricksworkspaceItem'; import { Helper } from '../../helpers/Helper'; +import { ThisExtension } from '../../ThisExtension'; export class DatabricksWorkspaceProviderItem implements vscode.FileStat, iDatabricksWorkspaceItem { // vscode.FileStat properties, basically all are read-only @@ -167,7 +168,9 @@ export class DatabricksWorkspaceProviderItem implements vscode.FileStat, iDatabr export class DatabricksWorkspaceProvider implements vscode.FileSystemProvider { constructor(context: vscode.ExtensionContext) { - context.subscriptions.push(vscode.workspace.registerFileSystemProvider('dbws', this, { isCaseSensitive: true })); + context.subscriptions.push(vscode.workspace.registerFileSystemProvider(ThisExtension.WORKSPACE_SCHEME, this, { isCaseSensitive: true })); + // legacy, to be removed in the future + context.subscriptions.push(vscode.workspace.registerFileSystemProvider(ThisExtension.WORKSPACE_SCHEME_LEGACY, this, { isCaseSensitive: true })); } // --- manage file metadata diff --git a/src/vscode/notebook/DatabricksKernel.ts b/src/vscode/notebook/DatabricksKernel.ts index 13a5fba..d0b2804 100644 --- a/src/vscode/notebook/DatabricksKernel.ts +++ b/src/vscode/notebook/DatabricksKernel.ts @@ -48,7 +48,7 @@ export class DatabricksKernel implements vscode.NotebookController { this.notebookType = notebookType; this._cluster = cluster; this._language = language; - this.id = DatabricksKernel.getId(cluster.cluster_id, notebookType); + this.id = DatabricksKernel.getId(this.KernelID, notebookType); this.label = DatabricksKernel.getLabel(cluster.cluster_name); this._executionOrder = 0; @@ -73,8 +73,8 @@ export class DatabricksKernel implements vscode.NotebookController { } async _onDidOpenNotebookDocument(notebook: vscode.NotebookDocument) { - // set this controller as recommended Kernel for notebooks opened via dbws:/ file system or from or local sync folder - if (notebook.uri.scheme == "dbws" || notebook.uri.toString().startsWith(ThisExtension.ActiveConnection.localSyncFolder.toString())) { + // set this controller as recommended Kernel for notebooks opened via dbws:/, wsfs:/ file system or from or local sync folder + if (notebook.uri.scheme == ThisExtension.WORKSPACE_SCHEME_LEGACY || notebook.uri.scheme == ThisExtension.WORKSPACE_SCHEME || notebook.uri.toString().startsWith(ThisExtension.ActiveConnection.localSyncFolder.toString())) { this.Controller.updateNotebookAffinity(notebook, vscode.NotebookControllerAffinity.Preferred); } @@ -106,8 +106,8 @@ export class DatabricksKernel implements vscode.NotebookController { } // #region Cluster-properties - static getId(clusterId: string, kernelType: KernelType) { - return this.baseId + clusterId + "-" + kernelType; + static getId(kernelId: string, kernelType: KernelType) { + return kernelId + "-" + kernelType; } static getLabel(clusterName: string) { @@ -118,6 +118,10 @@ export class DatabricksKernel implements vscode.NotebookController { return this._controller; } + get KernelID(): string { + return this._cluster.kernel_id ?? this._cluster.cluster_id; + } + get ClusterID(): string { return this._cluster.cluster_id; } @@ -387,9 +391,10 @@ export class DatabricksKernel implements vscode.NotebookController { } else { // absolute path provided switch (cell.notebook.uri.scheme) { - case "dbws": + case ThisExtension.WORKSPACE_SCHEME_LEGACY: // legacy + case ThisExtension.WORKSPACE_SCHEME: runUri = vscode.Uri.file(runFile); - runUri = runUri.with({ scheme: "dbws" }) + runUri = runUri.with({ scheme: ThisExtension.WORKSPACE_SCHEME }) break; case "file": runUri = await FSHelper.joinPath(ThisExtension.ActiveConnection.localSyncFolder, ThisExtension.ActiveConnection.localSyncSubfolders.Workspace, runFile); @@ -402,7 +407,8 @@ export class DatabricksKernel implements vscode.NotebookController { ThisExtension.log("Executing %run for '" + runUri + "' ..."); try { switch (runUri.scheme) { - case "dbws": + case ThisExtension.WORKSPACE_SCHEME_LEGACY: // legacy + case ThisExtension.WORKSPACE_SCHEME: if (!await FSHelper.pathExists(runUri)) { throw vscode.FileSystemError.FileNotFound(runUri); } diff --git a/src/vscode/notebook/DatabricksKernelManager.ts b/src/vscode/notebook/DatabricksKernelManager.ts index f80e899..5e45c78 100644 --- a/src/vscode/notebook/DatabricksKernelManager.ts +++ b/src/vscode/notebook/DatabricksKernelManager.ts @@ -58,7 +58,7 @@ export abstract class DatabricksKernelManager { } static getNotebookKernelName(cluster: iDatabricksCluster): string { - return cluster.cluster_id + DatabricksKernelManager.NotebookKernelSuffix; + return (cluster.kernel_id ?? cluster.cluster_id) + DatabricksKernelManager.NotebookKernelSuffix; } static getNotebookKernel(cluster: iDatabricksCluster): DatabricksKernel { diff --git a/src/vscode/treeviews/clusters/DatabricksCluster.ts b/src/vscode/treeviews/clusters/DatabricksCluster.ts index 760a67c..f0eb9d3 100644 --- a/src/vscode/treeviews/clusters/DatabricksCluster.ts +++ b/src/vscode/treeviews/clusters/DatabricksCluster.ts @@ -116,11 +116,6 @@ export class DatabricksCluster extends DatabricksClusterTreeItem { return FSHelper.joinPathSync(ThisExtension.rootUri, 'resources', theme, state + '.png'); } - readonly command = { - command: 'databricksClusterItem.click', title: "Open File", arguments: [this] - }; - - get definition(): iDatabricksCluster { return this._definition; } @@ -250,6 +245,14 @@ export class DatabricksCluster extends DatabricksClusterTreeItem { setTimeout(() => vscode.commands.executeCommand("databricksSQL.refresh", undefined, false), 1000); } + // set cluster for Databricks extension + async attachCluster(): Promise { + vscode.commands.executeCommand("databricks.connection.attachCluster", this.cluster_id); + ThisExtension.SQLClusterID = this.cluster_id; + + setTimeout(() => vscode.commands.executeCommand("databricksSQL.refresh", undefined, false), 1000); + } + async createKernel(logMessages: boolean = true): Promise { DatabricksKernelManager.createKernels(this.definition, logMessages); } diff --git a/src/vscode/treeviews/clusters/iDatabricksCluster.ts b/src/vscode/treeviews/clusters/iDatabricksCluster.ts index de93675..792a89a 100644 --- a/src/vscode/treeviews/clusters/iDatabricksCluster.ts +++ b/src/vscode/treeviews/clusters/iDatabricksCluster.ts @@ -1,6 +1,8 @@ import { ClusterState, ClusterSource } from './_types'; export interface iDatabricksCluster { + kernel_id?: string; // to overwrite the id of the kernel thats being created + cluster_id: string; cluster_name: string; state: ClusterState; diff --git a/src/vscode/treeviews/connections/DatabricksConnectionManagerAzure.ts b/src/vscode/treeviews/connections/DatabricksConnectionManagerAzure.ts index 0ea533a..3d8974f 100644 --- a/src/vscode/treeviews/connections/DatabricksConnectionManagerAzure.ts +++ b/src/vscode/treeviews/connections/DatabricksConnectionManagerAzure.ts @@ -7,6 +7,7 @@ import { DatabricksConnectionTreeItem } from './DatabricksConnectionTreeItem'; import { fetch, getProxyAgent, RequestInit, Response } from '@env/fetch'; import { AzureResourceListRepsonse, AzureResource, AzureSubscriptionListRepsonse } from './_types'; import { FSHelper } from '../../../helpers/FSHelper'; +import { DatabricksApiService } from '../../../databricksApi/databricksApiService'; interface AzureConfig { @@ -87,6 +88,9 @@ export class DatabricksConnectionManagerAzure extends DatabricksConnectionManage private static async _onDidChangeSessions(event: vscode.AuthenticationSessionsChangeEvent) { vscode.window.showWarningMessage("Session Changed! " + event.provider.id); + + let headers = await ThisExtension.ConnectionManager.getAuthorizationHeaders(ThisExtension.ActiveConnection); + DatabricksApiService.updateHeaders(headers); } private async getAADAccessToken(scopes: string[], tenantId?: string): Promise { diff --git a/src/vscode/treeviews/connections/DatabricksConnectionManagerDatabricks.ts b/src/vscode/treeviews/connections/DatabricksConnectionManagerDatabricks.ts new file mode 100644 index 0000000..0493e13 --- /dev/null +++ b/src/vscode/treeviews/connections/DatabricksConnectionManagerDatabricks.ts @@ -0,0 +1,111 @@ +import * as vscode from 'vscode'; + +import { ThisExtension } from '../../../ThisExtension'; +import { iDatabricksConnection } from './iDatabricksConnection'; +import { DatabricksConnectionManager } from './DatabricksConnectionManager'; +import { DatabricksKernelManager } from '../../notebook/DatabricksKernelManager'; +import { iDatabricksCluster } from '../clusters/iDatabricksCluster'; + +export class DatabricksConnectionManagerDatabricks extends DatabricksConnectionManager { + + private _databricksConnectionManager: any; + private _apiClient: any; + + constructor() { + super(); + } + + // TODO + /* + there is also an event that fires when the Databricks Extension config is changed + in that case we should also re-initialize the connection manager + */ + + async initialize(): Promise { + ThisExtension.log("Initializing ConnectionManager Databricks Extension ..."); + this._initialized = false; + + await this.loadConnections(); + + if (this._connections.length == 0) { + let msg: string = "No Databricks Workspaces have been found! Please make sure you are connected to the right tenant and have the appropriate permissions!"; + ThisExtension.log(msg); + vscode.window.showErrorMessage(msg); + } + else { + await super.manageLastActiveConnection(); + + try { + ThisExtension.log("Setting 'databricks.lastActiveConnection' to '" + this._lastActiveConnectionName + "' ..."); + ThisExtension.updateConfigurationSetting("databricks.lastActiveConnection", this._lastActiveConnectionName); + this._initialized = true; + + await this.activateConnection(this.LastActiveConnection, true); + + let cluster = this._databricksConnectionManager.cluster.details as iDatabricksCluster; + ThisExtension.SQLClusterID = cluster.cluster_id; + cluster.cluster_name = "Extension (Generic)"; + cluster.kernel_id = "databricks_extension_generic"; + + DatabricksKernelManager.createKernels(cluster); + } catch (error) { + let msg = "Could not activate Connection '" + this._lastActiveConnectionName + "'!"; + ThisExtension.log(msg); + vscode.window.showErrorMessage(msg); + } + } + } + + async loadConnections(): Promise { + try { + this._connections = []; + + const databricksExtension: vscode.Extension = vscode.extensions.getExtension("databricks.databricks"); + if (!databricksExtension) { + vscode.window.showErrorMessage("Please install the Databricks extension ('databricks.databricks') first!"); + // TODO + /* + You can trigger the installation using the workbench.extensions.installExtension command (see https://code.visualstudio.com/api/references/commands). + */ + return; + } + + ThisExtension.log("Databricks extension is installed!"); + let publicApi = await databricksExtension.activate(); + let connectionManager = publicApi.connectionManager; + await connectionManager.login(); + this._databricksConnectionManager = connectionManager;; + + let workspaceManager = connectionManager.workspaceClient; + this._apiClient = workspaceManager.apiClient; + let host = await this._apiClient.host; + + let localSyncfolder = connectionManager.syncDestinationMapper?.localUri; + + this._connections.push({ + "apiRootUrl": vscode.Uri.parse(host), + "displayName": "Databricks Extension", + "localSyncFolder": localSyncfolder.uri, + "exportFormats": { + "Scala": ".scala", + "Python": ".ipynb", + "SQL": ".sql", + "R": ".r" + }, + "useCodeCells": false, + "_source": "DatabricksExtension" + }) + } catch (e) { + ThisExtension.log(e); + } + } + + updateConnection(updatedCon: iDatabricksConnection): void { } + + async getAuthorizationHeaders(con: iDatabricksConnection): Promise { + const headers: Record = {}; + await this._apiClient.config.authenticate(headers); + + return headers + } +} \ No newline at end of file diff --git a/src/vscode/treeviews/connections/DatabricksConnectionTreeItem.ts b/src/vscode/treeviews/connections/DatabricksConnectionTreeItem.ts index 1e1feb9..834bb7d 100644 --- a/src/vscode/treeviews/connections/DatabricksConnectionTreeItem.ts +++ b/src/vscode/treeviews/connections/DatabricksConnectionTreeItem.ts @@ -55,7 +55,10 @@ export class DatabricksConnectionTreeItem extends vscode.TreeItem implements iDa this._isActive = false; - this.manageSecureToken(); + if(this._source != "DatabricksExtension") + { + this.manageSecureToken(); + } } //tooltip = this._tooltip; @@ -296,7 +299,6 @@ export class DatabricksConnectionTreeItem extends vscode.TreeItem implements iDa let uri: vscode.Uri = vscode.Uri.file(con.localSyncFolder); con.localSyncFolder = uri; } - catch { msg = 'Configuration ' + con.displayName + ': Property "localSyncFolder" is not a valid path! Please check your user and/or workspace settings!'; @@ -317,18 +319,21 @@ export class DatabricksConnectionTreeItem extends vscode.TreeItem implements iDa // check defaultvalues, etc. if (!this.propertyIsValid(con.exportFormats)) { let defaultFromExtension = ThisExtension.configuration.packageJSON.contributes.configuration[0].properties["databricks.connection.default.exportFormats"].default; - con.exportFormats = defaultFromExtension; msg = 'Configuration ' + con.displayName + ': Property "exportFormats" was not provided - using the default value!'; ThisExtension.log(msg); //vscode.window.showWarningMessage(msg); + + con.exportFormats = defaultFromExtension; } if (con.useCodeCells == undefined) { // this.propertyIsValid does not work for booleans !!! // get the default from the config of this extension let defaultFromExtension = ThisExtension.configuration.packageJSON.contributes.configuration[0].properties["databricks.connection.default.useCodeCells"].default; - con.useCodeCells = defaultFromExtension; msg = 'Configuration ' + con.displayName + ': Property "useCodeCells" was not provided - using the default value "' + defaultFromExtension + '"!'; ThisExtension.log(msg); //vscode.window.showWarningMessage(msg); + + con.useCodeCells = defaultFromExtension; + } return true; diff --git a/src/vscode/treeviews/connections/_types.ts b/src/vscode/treeviews/connections/_types.ts index 34056e9..01e9fd6 100644 --- a/src/vscode/treeviews/connections/_types.ts +++ b/src/vscode/treeviews/connections/_types.ts @@ -1,16 +1,27 @@ export type CloudProvider = "Azure" - | "AWS" - | "GCP" - ; +| "AWS" +| "GCP" +; + +export type ConnectionManager = + "Default" +| "VSCode Settings" +| "Databricks CLI Profiles" +| "Azure" +| "Databricks Extension" +| "Manual" +; export type ConnectionSource = "databricks.connections" - | "databricks.default" - | "CLI-profile" - | "ManuallyAdded" - | "Azure" - ; +| "databricks.default" +| "CLI-profile" +| "ManuallyAdded" +| "Azure" +| "DatabricksExtension" +; + export interface AccessTokenSecure { keyTarSettingName: string | undefined; diff --git a/src/vscode/treeviews/dbfs/DatabricksFSFile.ts b/src/vscode/treeviews/dbfs/DatabricksFSFile.ts index a888034..3dece20 100644 --- a/src/vscode/treeviews/dbfs/DatabricksFSFile.ts +++ b/src/vscode/treeviews/dbfs/DatabricksFSFile.ts @@ -222,6 +222,6 @@ export class DatabricksFSFile extends DatabricksFSTreeItem { } async compare(): Promise { - Helper.showDiff(vscode.Uri.parse("dbfs:/" + this.path), this.localPath); + Helper.showDiff(vscode.Uri.parse(ThisExtension.DBFS_SCHEME + ":/" + this.path), this.localPath); } } \ No newline at end of file diff --git a/src/vscode/treeviews/dbfs/DatabricksFSTreeItem.ts b/src/vscode/treeviews/dbfs/DatabricksFSTreeItem.ts index 5df0eae..d01d12f 100644 --- a/src/vscode/treeviews/dbfs/DatabricksFSTreeItem.ts +++ b/src/vscode/treeviews/dbfs/DatabricksFSTreeItem.ts @@ -95,7 +95,7 @@ export class DatabricksFSTreeItem extends vscode.TreeItem implements iDatabricks } get dbfsUri(): vscode.Uri { - return vscode.Uri.parse("dbfs:" + this.path); + return vscode.Uri.parse(ThisExtension.DBFS_SCHEME + ":" + this.path); } public static fromInterface(item: iDatabricksFSItem, parent: DatabricksFSDirectory): DatabricksFSTreeItem { diff --git a/src/vscode/treeviews/sql/DatabricksSQLSelectCluster.ts b/src/vscode/treeviews/sql/DatabricksSQLSelectCluster.ts deleted file mode 100644 index 0b233a0..0000000 --- a/src/vscode/treeviews/sql/DatabricksSQLSelectCluster.ts +++ /dev/null @@ -1,16 +0,0 @@ -import * as vscode from 'vscode'; - -import { DatabricksSQLTreeItem } from './DatabricksSQLTreeItem'; -import { SQLItemType } from './_types'; - -// https://vshaxe.github.io/vscode-extern/vscode/TreeItem.html -export class DatabricksSQLSelectCluster extends DatabricksSQLTreeItem { - - constructor() { - super("Please select a cluster first!", undefined, undefined, undefined, vscode.TreeItemCollapsibleState.None); - } - - get itemType(): SQLItemType { - return 'DATABASE'; - } -} \ No newline at end of file diff --git a/src/vscode/treeviews/sql/DatabricksSQLTreeProvider.ts b/src/vscode/treeviews/sql/DatabricksSQLTreeProvider.ts index 32cba05..731df8a 100644 --- a/src/vscode/treeviews/sql/DatabricksSQLTreeProvider.ts +++ b/src/vscode/treeviews/sql/DatabricksSQLTreeProvider.ts @@ -5,7 +5,6 @@ import { Helper } from '../../../helpers/Helper'; import { DatabricksApiService } from '../../../databricksApi/databricksApiService'; import { ThisExtension } from '../../../ThisExtension'; import { DatabricksSQLDatabase } from './DatabricksSQLDatabase'; -import { DatabricksSQLSelectCluster } from './DatabricksSQLSelectCluster'; import { DatabricksSQLTreeItem } from './DatabricksSQLTreeItem'; import { iDatabricksApiCommandsStatusResponse } from '../../../databricksApi/_types'; @@ -59,7 +58,7 @@ export class DatabricksSQLTreeProvider implements vscode.TreeDataProvider { - let onlineFileTempPath: vscode.Uri = vscode.Uri.parse("dbws:" + this.path); + let onlineFileTempPath: vscode.Uri = vscode.Uri.parse(ThisExtension.WORKSPACE_SCHEME + ":" + this.path); Helper.showDiff(onlineFileTempPath, this.localPath); } diff --git a/src/vscode/treeviews/workspaces/DatabricksWorkspaceNotebook.ts b/src/vscode/treeviews/workspaces/DatabricksWorkspaceNotebook.ts index 8c530c2..638d87b 100644 --- a/src/vscode/treeviews/workspaces/DatabricksWorkspaceNotebook.ts +++ b/src/vscode/treeviews/workspaces/DatabricksWorkspaceNotebook.ts @@ -241,7 +241,7 @@ export class DatabricksWorkspaceNotebook extends DatabricksWorkspaceTreeItem { } async compare(): Promise { - let onlineFileTempPath: vscode.Uri = vscode.Uri.parse("dbws:" + this.path); + let onlineFileTempPath: vscode.Uri = vscode.Uri.parse(ThisExtension.WORKSPACE_SCHEME + ":" + this.path); Helper.showDiff(onlineFileTempPath, this.localPath); }