From acad0db81a626df15ced0f0dcc2a36835369d9b4 Mon Sep 17 00:00:00 2001 From: Gerhard Brueckl Date: Sat, 18 Feb 2023 21:24:50 +0100 Subject: [PATCH 1/9] integration with official Databricks Extension --- CHANGELOG.md | 6 ++ README.md | 5 + package.json | 38 ++++++-- resources/databricks_sidebar_orig.svg | 1 + src/ThisExtension.ts | 30 +++++- src/extension.ts | 5 +- .../treeviews/clusters/DatabricksCluster.ts | 8 ++ .../DatabricksConnectionManagerDatabricks.ts | 92 +++++++++++++++++++ .../DatabricksConnectionTreeItem.ts | 13 ++- src/vscode/treeviews/connections/_types.ts | 26 ++++-- .../sql/DatabricksSQLSelectCluster.ts | 16 ---- .../sql/DatabricksSQLTreeProvider.ts | 3 +- 12 files changed, 198 insertions(+), 45 deletions(-) create mode 100644 resources/databricks_sidebar_orig.svg create mode 100644 src/vscode/treeviews/connections/DatabricksConnectionManagerDatabricks.ts delete mode 100644 src/vscode/treeviews/sql/DatabricksSQLSelectCluster.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index d79991f..b72dc3f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Release Notes +**v1.9.0:** +- added integration with [official Databricks extensions](https://marketplace.visualstudio.com/items?itemName=databricks.databricks) + - new connection manager [Databricks Extensions](README.md/#setup-and-configuration-databricks-extension-connection-manager) + - derive cluster for [SQL Browser](README.md/#sql-browser) + - change cluster using [Cluster Manager](README.md/#cluster-manager) + **v1.5.0:** - added support for [Widgets](README.md/#widgets) when running Notebooks diff --git a/README.md b/README.md index a81051b..9990edd 100644 --- a/README.md +++ b/README.md @@ -169,6 +169,10 @@ The following Azure-specific settings exist and can be set in the workspace sett They are documented via VSCode settings documentation. +# Setup and Configuration (Databricks Extension Connection Manager) +This connection manager leverages the [official Databricks extensions](https://marketplace.visualstudio.com/items?itemName=databricks.databricks) to establish a connection with your Databricks workspace. It only supports a single connection hence the actual Connection Manager tab will be hidden for this connection manager. +It also derives the cluster automatically from the Databricks extensions to source the [SQL Browser](#sql-browser) but also allows you to change it directly from the [Cluster Manager](#cluster-manager) using the `Attach cluster` command. + # Connection Manager ![Connection Manager](/images/ConnectionManager.jpg?raw=true "Connection Manager") @@ -176,6 +180,7 @@ The extension supports various connection managers and the list can be easily ex - [VSCode Settings](#setup-and-configuration-vscode-connection-manager) - [Databricks CLI](#setup-and-configuration-databricks-cli-connection-manager) - [Azure](#setup-and-configuration-azure-connection-manager) +- [Databricks Extensions](#setup-and-configuration-databricks-extensions-connection-manager) - `Manual` where you are simply prompted to enter connection information at the start of your session. You can specify the one to use by setting the VSCode setting `databricks.connectionManager`. diff --git a/package.json b/package.json index 69a8a42..6855f92 100644 --- a/package.json +++ b/package.json @@ -1,8 +1,8 @@ { "name": "databricks-vscode", - "displayName": "Databricks VSCode", + "displayName": "Databricks Power Tools", "description": "Run notebooks cell-by-cell, browse and edit your Databricks Workspace, DBFS, Clusters, Jobs, Secrets, Repos and SQL. Supports Azure Databricks, Databricks on AWS and Databricks on GCP.", - "version": "1.5.1", + "version": "1.9.0", "publisher": "paiqo", "icon": "resources/databricks_extension.png", "author": { @@ -78,7 +78,8 @@ "VSCode Settings", "Databricks CLI Profiles", "Azure", - "ManualInput" + "Databricks Extension", + "Manual Input" ], "default": "VSCode Settings", "enumDescriptions": [ @@ -415,19 +416,20 @@ "viewsContainers": { "activitybar": [ { - "id": "databricks", + "id": "databricksBar", "title": "Databricks", - "icon": "resources/databricks_sidebar.png" + "icon": "resources/databricks_sidebar_orig.svg" } ] }, "views": { - "databricks": [ + "databricksBar": [ { "id": "databricksConnections", "name": "Connections", "type": "tree", - "initialSize": 25 + "initialSize": 25, + "when": "!paiqo.databricks.hideConnectionManager" }, { "id": "databricksWorkspace", @@ -468,6 +470,12 @@ } ] }, + "viewsWelcome": [ + { + "view": "databricksSQL", + "contents": "Please select a cluster first" + } + ], "commands": [ { "command": "databricksConnections.refresh", @@ -602,7 +610,14 @@ "icon": { "light": "resources/light/stop.png", "dark": "resources/dark/stop.png" - } + }, + "enablement": "paiqo.databricks.connectionManager != 'Databricks Extension'" + }, + { + "command": "databricksClusterItem.attachCluster", + "title": "Attach cluster", + "icon": "$(plug)", + "enablement": "paiqo.databricks.connectionManager == 'Databricks Extension'" }, { "command": "databricksClusterItem.createKernel", @@ -942,7 +957,12 @@ }, { "command": "databricksClusterItem.useForSQL", - "when": "view == databricksClusters && viewItem =~ /.*,STARTED,.*/ && viewItem =~ /.*,MANUAL,.*/" + "when": "view == databricksClusters && viewItem =~ /.*,STARTED,.*/ && viewItem =~ /.*,MANUAL,.*/ && paiqo.databricks.connectionManager != 'Databricks Extension'" + }, + { + "command": "databricksClusterItem.attachCluster", + "when": "view == databricksClusters && viewItem =~ /.*,MANUAL,.*/ && paiqo.databricks.connectionManager == 'Databricks Extension'", + "group": "inline" }, { "command": "databricksClusterItem.createKernel", diff --git a/resources/databricks_sidebar_orig.svg b/resources/databricks_sidebar_orig.svg new file mode 100644 index 0000000..3fd304f --- /dev/null +++ b/resources/databricks_sidebar_orig.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/src/ThisExtension.ts b/src/ThisExtension.ts index bc213dd..3241eb7 100644 --- a/src/ThisExtension.ts +++ b/src/ThisExtension.ts @@ -4,10 +4,11 @@ import { WorkspaceItemLanguage } from './vscode/treeviews/workspaces/_types'; import { DatabricksConnectionManager } from './vscode/treeviews/connections/DatabricksConnectionManager'; import { iDatabricksConnection } from './vscode/treeviews/connections/iDatabricksConnection'; import { DatabricksConnectionManagerVSCode } from './vscode/treeviews/connections/DatabricksConnectionManagerVSCode'; -import { SensitiveValueStore } from './vscode/treeviews/connections/_types'; +import { ConnectionManager, ConnectionSource, SensitiveValueStore } from './vscode/treeviews/connections/_types'; import { DatabricksConnectionManagerCLI } from './vscode/treeviews/connections/DatabricksConnectionManagerCLI'; import { DatabricksConnectionManagerManualInput } from './vscode/treeviews/connections/DatabricksConnectionManagerManualInput'; import { DatabricksConnectionManagerAzure } from './vscode/treeviews/connections/DatabricksConnectionManagerAzure'; +import { DatabricksConnectionManagerDatabricks } from './vscode/treeviews/connections/DatabricksConnectionManagerDatabricks'; // https://vshaxe.github.io/vscode-extern/vscode/TreeDataProvider.html export abstract class ThisExtension { @@ -17,6 +18,7 @@ export abstract class ThisExtension { private static _statusBar: vscode.StatusBarItem; private static _isValidated: boolean = false; private static _logger: vscode.OutputChannel; + private static _connectionManagerText: ConnectionManager; private static _connectionManager: DatabricksConnectionManager; private static _settingScope: ConfigSettingSource; private static _sensitiveValueStore: SensitiveValueStore; @@ -61,7 +63,7 @@ export abstract class ThisExtension { ThisExtension.readGlobalSettings(); - let connectionManager = this.getConfigurationSetting("databricks.connectionManager", this.SettingScope, true); + let connectionManager = this.getConfigurationSetting("databricks.connectionManager", this.SettingScope, true); switch (connectionManager.value) { case "VSCode Settings": this._connectionManager = new DatabricksConnectionManagerVSCode(); @@ -72,13 +74,29 @@ export abstract class ThisExtension { case "Azure": this._connectionManager = new DatabricksConnectionManagerAzure(); break; - case "ManualInput": + case "Databricks Extension": + this._connectionManager = new DatabricksConnectionManagerDatabricks(); + // we hide the Connections Tab as we load all information from the Databricks Extension + vscode.commands.executeCommand( + "setContext", + "paiqo.databricks.hideConnectionManager", + true + ); + break; + case "Manual Input": this._connectionManager = new DatabricksConnectionManagerManualInput(); break; default: this.log("'" + connectionManager + "' is not a valid value for config setting 'databricks.connectionManager!"); - + } + this._connectionManagerText = connectionManager.value as ConnectionManager; + + vscode.commands.executeCommand( + "setContext", + "paiqo.databricks.connectionManager", + this._connectionManagerText + ); await this.ConnectionManager.initialize(); @@ -126,6 +144,10 @@ export abstract class ThisExtension { return this._connectionManager; } + static get ConnectionManagerText(): ConnectionManager { + return this._connectionManagerText; + } + static get SQLClusterID(): string { return this._sqlClusterId; } diff --git a/src/extension.ts b/src/extension.ts index eb97ea5..54f7d2f 100644 --- a/src/extension.ts +++ b/src/extension.ts @@ -69,7 +69,6 @@ export async function activate(context: vscode.ExtensionContext) { ); ThisExtension.setStatusBar("Kernels initialized!"); - // register DatabricksConnectionTreeProvider let databricksConnectionTreeProvider = new DatabricksConnectionTreeProvider(); vscode.window.registerTreeDataProvider('databricksConnections', databricksConnectionTreeProvider); @@ -82,7 +81,6 @@ export async function activate(context: vscode.ExtensionContext) { } vscode.commands.registerCommand('databricksConnectionItem.activate', (connection: DatabricksConnectionTreeItem) => connection.activate()); - // register DatabricksWorkspaceTreeProvider if (!ThisExtension.isInBrowser) { let databricksWorkspaceTreeProvider = new DatabricksWorkspaceTreeProvider(context); @@ -127,6 +125,9 @@ export async function activate(context: vscode.ExtensionContext) { vscode.commands.registerCommand('databricksClusterItem.createKernel', (cluster: DatabricksCluster) => cluster.createKernel()); vscode.commands.registerCommand('databricksClusterItem.restartKernel', (cluster: DatabricksCluster) => cluster.restartKernel()); + // set cluster for Databricks Extension + vscode.commands.registerCommand('databricksClusterItem.attachCluster', (cluster: DatabricksCluster) => cluster.attachCluster()); + // register DatabricksJobsTreeProvider let databricksJobsTreeProvider = new DatabricksJobTreeProvider(context); diff --git a/src/vscode/treeviews/clusters/DatabricksCluster.ts b/src/vscode/treeviews/clusters/DatabricksCluster.ts index 760a67c..996ad79 100644 --- a/src/vscode/treeviews/clusters/DatabricksCluster.ts +++ b/src/vscode/treeviews/clusters/DatabricksCluster.ts @@ -250,6 +250,14 @@ export class DatabricksCluster extends DatabricksClusterTreeItem { setTimeout(() => vscode.commands.executeCommand("databricksSQL.refresh", undefined, false), 1000); } + // set cluster for Databricks extension + async attachCluster(): Promise { + vscode.commands.executeCommand("databricks.connection.attachCluster", this.cluster_id); + ThisExtension.SQLClusterID = this.cluster_id; + + setTimeout(() => vscode.commands.executeCommand("databricksSQL.refresh", undefined, false), 1000); + } + async createKernel(logMessages: boolean = true): Promise { DatabricksKernelManager.createKernels(this.definition, logMessages); } diff --git a/src/vscode/treeviews/connections/DatabricksConnectionManagerDatabricks.ts b/src/vscode/treeviews/connections/DatabricksConnectionManagerDatabricks.ts new file mode 100644 index 0000000..2b16651 --- /dev/null +++ b/src/vscode/treeviews/connections/DatabricksConnectionManagerDatabricks.ts @@ -0,0 +1,92 @@ +import * as vscode from 'vscode'; + +import { ThisExtension } from '../../../ThisExtension'; +import { iDatabricksConnection } from './iDatabricksConnection'; +import { DatabricksConnectionManager } from './DatabricksConnectionManager'; + +export class DatabricksConnectionManagerDatabricks extends DatabricksConnectionManager { + + private _databricksConnectionManager: any; + + constructor() { + super(); + } + + async initialize(): Promise { + ThisExtension.log("Initializing ConnectionManager Databricks Extension ..."); + this._initialized = false; + + await this.loadConnections(); + + if (this._connections.length == 0) { + let msg: string = "No Databricks Workspaces have been found! Please make sure you are connected to the right tenant and have the appropriate permissions!"; + ThisExtension.log(msg); + vscode.window.showErrorMessage(msg); + } + else { + await super.manageLastActiveConnection(); + + try { + ThisExtension.log("Setting 'databricks.lastActiveConnection' to '" + this._lastActiveConnectionName + "' ..."); + ThisExtension.updateConfigurationSetting("databricks.lastActiveConnection", this._lastActiveConnectionName); + this._initialized = true; + + await this.activateConnection(this.LastActiveConnection, true); + + ThisExtension.SQLClusterID = this._databricksConnectionManager.cluster.id; + + } catch (error) { + let msg = "Could not activate Connection '" + this._lastActiveConnectionName + "'!"; + ThisExtension.log(msg); + vscode.window.showErrorMessage(msg); + } + } + } + + async loadConnections(): Promise { + try { + this._connections = []; + + const databricksExtension: vscode.Extension = vscode.extensions.getExtension("databricks.databricks"); + if (!databricksExtension) { + vscode.window.showErrorMessage("Please install the Databricks extension ('databricks.databricks') first!"); + return; + } + + ThisExtension.log("Databricks extension is installed!"); + let publicApi = await databricksExtension.activate(); + let connectionManager = publicApi.connectionManager; + await connectionManager.login(); + this._databricksConnectionManager = connectionManager;; + + let workspaceManager = connectionManager.workspaceClient; + let apiClient = workspaceManager.apiClient; + let host = await apiClient.host; + let token = apiClient.config.token; + + let localSyncfolder = connectionManager.syncDestinationMapper?.localUri; + + this._connections.push({ + "apiRootUrl": vscode.Uri.parse(host), + "personalAccessToken": token, + "displayName": "Databricks Extension", + "localSyncFolder": localSyncfolder.uri, + "exportFormats": { + "Scala": ".scala", + "Python": ".ipynb", + "SQL": ".sql", + "R": ".r" + }, + "useCodeCells": false, + "_source": "DatabricksExtension" + }) + + let i = 0; + + } catch (e) { + ThisExtension.log(e); + } + } + + updateConnection(updatedCon: iDatabricksConnection): void { } +} \ No newline at end of file diff --git a/src/vscode/treeviews/connections/DatabricksConnectionTreeItem.ts b/src/vscode/treeviews/connections/DatabricksConnectionTreeItem.ts index 1e1feb9..834bb7d 100644 --- a/src/vscode/treeviews/connections/DatabricksConnectionTreeItem.ts +++ b/src/vscode/treeviews/connections/DatabricksConnectionTreeItem.ts @@ -55,7 +55,10 @@ export class DatabricksConnectionTreeItem extends vscode.TreeItem implements iDa this._isActive = false; - this.manageSecureToken(); + if(this._source != "DatabricksExtension") + { + this.manageSecureToken(); + } } //tooltip = this._tooltip; @@ -296,7 +299,6 @@ export class DatabricksConnectionTreeItem extends vscode.TreeItem implements iDa let uri: vscode.Uri = vscode.Uri.file(con.localSyncFolder); con.localSyncFolder = uri; } - catch { msg = 'Configuration ' + con.displayName + ': Property "localSyncFolder" is not a valid path! Please check your user and/or workspace settings!'; @@ -317,18 +319,21 @@ export class DatabricksConnectionTreeItem extends vscode.TreeItem implements iDa // check defaultvalues, etc. if (!this.propertyIsValid(con.exportFormats)) { let defaultFromExtension = ThisExtension.configuration.packageJSON.contributes.configuration[0].properties["databricks.connection.default.exportFormats"].default; - con.exportFormats = defaultFromExtension; msg = 'Configuration ' + con.displayName + ': Property "exportFormats" was not provided - using the default value!'; ThisExtension.log(msg); //vscode.window.showWarningMessage(msg); + + con.exportFormats = defaultFromExtension; } if (con.useCodeCells == undefined) { // this.propertyIsValid does not work for booleans !!! // get the default from the config of this extension let defaultFromExtension = ThisExtension.configuration.packageJSON.contributes.configuration[0].properties["databricks.connection.default.useCodeCells"].default; - con.useCodeCells = defaultFromExtension; msg = 'Configuration ' + con.displayName + ': Property "useCodeCells" was not provided - using the default value "' + defaultFromExtension + '"!'; ThisExtension.log(msg); //vscode.window.showWarningMessage(msg); + + con.useCodeCells = defaultFromExtension; + } return true; diff --git a/src/vscode/treeviews/connections/_types.ts b/src/vscode/treeviews/connections/_types.ts index 34056e9..1ab7821 100644 --- a/src/vscode/treeviews/connections/_types.ts +++ b/src/vscode/treeviews/connections/_types.ts @@ -1,16 +1,26 @@ export type CloudProvider = "Azure" - | "AWS" - | "GCP" - ; +| "AWS" +| "GCP" +; + +export type ConnectionManager = + "VSCode Settings" +| "Databricks CLI Profiles" +| "Azure" +| "Databricks Extension" +| "Manual Input" +; export type ConnectionSource = "databricks.connections" - | "databricks.default" - | "CLI-profile" - | "ManuallyAdded" - | "Azure" - ; +| "databricks.default" +| "CLI-profile" +| "ManuallyAdded" +| "Azure" +| "DatabricksExtension" +; + export interface AccessTokenSecure { keyTarSettingName: string | undefined; diff --git a/src/vscode/treeviews/sql/DatabricksSQLSelectCluster.ts b/src/vscode/treeviews/sql/DatabricksSQLSelectCluster.ts deleted file mode 100644 index 0b233a0..0000000 --- a/src/vscode/treeviews/sql/DatabricksSQLSelectCluster.ts +++ /dev/null @@ -1,16 +0,0 @@ -import * as vscode from 'vscode'; - -import { DatabricksSQLTreeItem } from './DatabricksSQLTreeItem'; -import { SQLItemType } from './_types'; - -// https://vshaxe.github.io/vscode-extern/vscode/TreeItem.html -export class DatabricksSQLSelectCluster extends DatabricksSQLTreeItem { - - constructor() { - super("Please select a cluster first!", undefined, undefined, undefined, vscode.TreeItemCollapsibleState.None); - } - - get itemType(): SQLItemType { - return 'DATABASE'; - } -} \ No newline at end of file diff --git a/src/vscode/treeviews/sql/DatabricksSQLTreeProvider.ts b/src/vscode/treeviews/sql/DatabricksSQLTreeProvider.ts index 32cba05..731df8a 100644 --- a/src/vscode/treeviews/sql/DatabricksSQLTreeProvider.ts +++ b/src/vscode/treeviews/sql/DatabricksSQLTreeProvider.ts @@ -5,7 +5,6 @@ import { Helper } from '../../../helpers/Helper'; import { DatabricksApiService } from '../../../databricksApi/databricksApiService'; import { ThisExtension } from '../../../ThisExtension'; import { DatabricksSQLDatabase } from './DatabricksSQLDatabase'; -import { DatabricksSQLSelectCluster } from './DatabricksSQLSelectCluster'; import { DatabricksSQLTreeItem } from './DatabricksSQLTreeItem'; import { iDatabricksApiCommandsStatusResponse } from '../../../databricksApi/_types'; @@ -59,7 +58,7 @@ export class DatabricksSQLTreeProvider implements vscode.TreeDataProvider Date: Sat, 18 Feb 2023 22:12:24 +0100 Subject: [PATCH 2/9] add Default connection manager update readme --- package.json | 43 +++++++++++----------- src/ThisExtension.ts | 34 ++++++++++++++--- src/vscode/treeviews/connections/_types.ts | 3 +- 3 files changed, 52 insertions(+), 28 deletions(-) diff --git a/package.json b/package.json index 6855f92..6ecc500 100644 --- a/package.json +++ b/package.json @@ -68,24 +68,27 @@ "contributes": { "configuration": [ { - "name": "Databricks", + "name": "paiqo.databricks-vscode", "type": "object", - "title": "Databricks", + "title": "Databricks Power Tools", "properties": { "databricks.connectionManager": { "type": "string", "enum": [ + "Default", "VSCode Settings", "Databricks CLI Profiles", "Azure", "Databricks Extension", "Manual Input" ], - "default": "VSCode Settings", + "default": "Default", "enumDescriptions": [ + "The best matching connection manager is selected automatically.", "All connection information is stored securely in the VSCode Settings (user or workspace).", "Connection information is read from the Databricks CLI config file. Databricks CLI has to be configured upfront! Also supports the environment varialbe 'DATABRICKS_CONFIG_FILE'.", "Connection information is loaded from Azure. All Azure Databricks workspaces that you have access to will show up atuomatically in the connection list.", + "A single connection that is managed by the Databricks Extensions", "You are prompted for connection information once you interact with the extension." ], "description": "Defines how connection information is managed and stored.", @@ -95,16 +98,14 @@ "type": "string", "enum": [ "VSCodeSettings", - "SystemKeyChain", - "ExternalConfigFile" + "SystemKeyChain" ], "default": "SystemKeyChain", "enumDescriptions": [ "Databricks Personal Access Token is stored in the VSCode Settings as plain text! There is a potential risk that the PAT gets checked in to source control systems like git!", - "Databricks Personal Access Token is securely stored in the System Key Chain. Easy Copying/Sharing connections with other users is not possible this way. VSCode only stores a reference to the the Systems KeyChain entry.", - "(Experimental!) Databricks Personal Access Token is stored in an external Config file. This is useful to manage all your PATs in one place and also easily share them." + "Databricks Personal Access Token is securely stored in the System Key Chain. Easy Copying/Sharing connections with other users is not possible this way. VSCode only stores a reference to the the Systems KeyChain entry." ], - "description": "Defines the default how the VSCode extension for Databricks stores sensitive values like the Databricks Personal Access Token (PAT). Only releveant if the VSCode Connection Manager is used!", + "markdownDescription": "(`VSCode Settings`) Defines the default how the VSCode extension for Databricks stores sensitive values like the Databricks Personal Access Token (PAT). Only releveant if the VSCode Connection Manager is used!", "scope": "window" }, "databricks.connection.default.exportFormats": { @@ -115,7 +116,7 @@ "SQL": ".sql", "R": ".r" }, - "description": "(Optional) The different file extension used when downloading notebooks from Databricks.", + "markdownDescription": "(`VSCode Settings`, Optional) The different file extension used when downloading notebooks from Databricks.", "scope": "window", "properties": { "Scala": { @@ -152,7 +153,7 @@ }, "databricks.connection.default.displayName": { "type": "string", - "description": "(Mandatory) Name shown in the Connection list of the Databricks VSCode extension. Must be unique across all VSCode workspaces to avoid conflicts!", + "markdownDescription": "(`VSCode Settings`, Mandatory) Name shown in the Connection list of the Databricks VSCode extension. Must be unique across all VSCode workspaces to avoid conflicts!", "scope": "window", "examples": [ "my Connection", @@ -162,7 +163,7 @@ }, "databricks.connection.default.apiRootUrl": { "type": "string", - "description": "(Mandatory) Root URL of the Databricks API. Azure: https://adb-12345678901234.19.azuredatabricks.net - AWS: https://abc-12345-xaz.cloud.databricks.com. Please do not provide any path or querystring like `?o=123456`", + "markdownDescription": "(`VSCode Settings`, Mandatory) Root URL of the Databricks API. Azure: https://adb-12345678901234.19.azuredatabricks.net - AWS: https://abc-12345-xaz.cloud.databricks.com. Please do not provide any path or querystring like `?o=123456`", "scope": "window", "examples": [ "https://adb-12345678901234.19.azuredatabricks.net/", @@ -171,7 +172,7 @@ }, "databricks.connection.default.personalAccessToken": { "type": "string", - "description": "(Mandatory) The Personal Access Token (PAT) used to access the Databricks API.", + "markdownDescription": "(`VSCode Settings`, Mandatory) The Personal Access Token (PAT) used to access the Databricks API.", "scope": "window", "examples": [ "dapi0123456789abcdef0123456789abcdef" @@ -179,7 +180,7 @@ }, "databricks.connection.default.personalAccessTokenSecure": { "type": "object", - "description": "(Optional) Reference to a secure store for the Personal Access Token.", + "markdownDescription": "(`VSCode Settings`, Optional) Reference to a secure store for the Personal Access Token.", "scope": "window", "maxProperties": 1, "properties": { @@ -197,7 +198,7 @@ }, "databricks.connection.default.localSyncFolder": { "type": "string", - "description": "(Mandatory) A local path where files (notebooks) that are downloaded from Databricks should be stored. This folder can then be integrated in your GIT repository. \nExamples: \nWindows - C:\\mySyncFolder \nLinux/Unix - /home/myUser/mySyncFolder \nmacOS - myUser/mySyncFolder (Please do not use leading / on macOS!)", + "markdownDescription": "(`VSCode Settings`, Mandatory) A local path where files (notebooks) that are downloaded from Databricks should be stored. This folder can then be integrated in your GIT repository. \nExamples: \nWindows - C:\\mySyncFolder \nLinux/Unix - /home/myUser/mySyncFolder \nmacOS - myUser/mySyncFolder (Please do not use leading / on macOS!)", "scope": "window", "examples": [ "Windows: C:\\mySyncFolder", @@ -213,7 +214,7 @@ "DBFS": "DBFS", "Jobs": "Jobs" }, - "description": "(Optional) Can be used to specify different sub-paths for the items that can be downloaded from Databricks.", + "markdownDescription": "(`VSCode Settings`, Optional) Can be used to specify different sub-paths for the items that can be downloaded from Databricks.", "scope": "window", "properties": { "Workspace": { @@ -236,13 +237,13 @@ }, "databricks.connection.default.useCodeCells": { "type": "boolean", - "description": "If true, Code Cell tags (# %%) will be added when downloading raw source files.", + "markdownDescription": "(`VSCode Settings`, Optional) If true, Code Cell tags (# %%) will be added when downloading raw source files.", "scope": "window", "default": true }, "databricks.connections": { "type": "array", - "description": "An array of objects where each object represents a Databricks Connection containing the same settings as defined for databricks.connection.default.* ", + "makrdownDescription": "(`VSCode Settings`) An array of objects where each object represents a Databricks Connection containing the same settings as defined for databricks.connection.default.* ", "items": { "type": "object", "properties": { @@ -365,17 +366,17 @@ }, "databricks.lastActiveConnection": { "type": "string", - "description": "(Optional) The displayName of the Connection that was used in the last session.", + "description": "The displayName of the Connection that was used in the last session.", "scope": "window" }, "databricks.azure.tenantId": { "type": "string", - "description": "(Optional) To connect to Azure Databricks Workspaces outside of your home-tenant.", + "markdownDescription": "(`Azure`, Optional) To connect to Azure Databricks Workspaces outside of your home-tenant.", "scope": "window" }, "databricks.azure.subscriptionIds": { "type": "array", - "description": "(Optional) A list of Azure Subscription IDs (GUIDs) from which the Databricks Workspaces are loaded. If not specified, all accessible subscriptions are searched.", + "markdownDescription": "(`Azure`, Optional) A list of Azure Subscription IDs (GUIDs) from which the Databricks Workspaces are loaded. If not specified, all accessible subscriptions are searched.", "scope": "window", "items": { "type": "string", @@ -387,7 +388,7 @@ }, "databricks.azure.workspaces": { "type": "array", - "description": "(Optional) A list of Azure Resource IDs to load directly without browsing the tenant first.", + "markdownDescription": "(`Azure`, Optional) A list of Azure Resource IDs to load directly without browsing the tenant first.", "scope": "window", "items": { "type": "object", diff --git a/src/ThisExtension.ts b/src/ThisExtension.ts index 3241eb7..a2a98c1 100644 --- a/src/ThisExtension.ts +++ b/src/ThisExtension.ts @@ -63,8 +63,28 @@ export abstract class ThisExtension { ThisExtension.readGlobalSettings(); + this._connectionManagerText = undefined; let connectionManager = this.getConfigurationSetting("databricks.connectionManager", this.SettingScope, true); - switch (connectionManager.value) { + let conManager: ConnectionManager = connectionManager.value; + + // handle default + if (conManager == "Default") { + ThisExtension.log("Default connection manager selected. Trying to find the best connection manager ...") + if (vscode.extensions.getExtension("databricks.databricks123")) { + ThisExtension.log("Databricks Extension found. Using it as connection manager ..."); + conManager = "Databricks Extension"; + } + else + { + ThisExtension.log("Databricks Extension not found. Using VSCode Settings as connection manager ..."); + conManager = "VSCode Settings"; + // should open workspace settings with a filter but the filter is not yet working + vscode.commands.executeCommand("workbench.action.openWorkspaceSettings", ThisExtension.configuration.id); + } + } + + switch (conManager) { + case "VSCode Settings": this._connectionManager = new DatabricksConnectionManagerVSCode(); break; @@ -90,13 +110,15 @@ export abstract class ThisExtension { this.log("'" + connectionManager + "' is not a valid value for config setting 'databricks.connectionManager!"); } - this._connectionManagerText = connectionManager.value as ConnectionManager; + if (!this._connectionManagerText) { + this._connectionManagerText = connectionManager.value as ConnectionManager; + } vscode.commands.executeCommand( - "setContext", - "paiqo.databricks.connectionManager", - this._connectionManagerText - ); + "setContext", + "paiqo.databricks.connectionManager", + this._connectionManagerText + ); await this.ConnectionManager.initialize(); diff --git a/src/vscode/treeviews/connections/_types.ts b/src/vscode/treeviews/connections/_types.ts index 1ab7821..b7788ea 100644 --- a/src/vscode/treeviews/connections/_types.ts +++ b/src/vscode/treeviews/connections/_types.ts @@ -5,7 +5,8 @@ export type CloudProvider = ; export type ConnectionManager = - "VSCode Settings" + "Default" +| "VSCode Settings" | "Databricks CLI Profiles" | "Azure" | "Databricks Extension" From c1464bd10b7802fe70e6a970650f91ec85d303aa Mon Sep 17 00:00:00 2001 From: Gerhard Brueckl Date: Mon, 20 Feb 2023 13:43:25 +0100 Subject: [PATCH 3/9] change File System provider to also support `wsfs:/` which will probably replace `dbws:/` in the future --- CHANGELOG.md | 2 +- README.md | 7 ++- package.json | 5 +- src/ThisExtension.ts | 5 ++ src/extension.ts | 8 +-- src/helpers/Helper.ts | 62 +++++++++++++++---- .../DatabricksFileSystemProvider.ts | 3 +- .../DatabricksWorkspaceProvider.ts | 2 + src/vscode/notebook/DatabricksKernel.ts | 12 ++-- .../treeviews/dbfs/DatabricksFSTreeItem.ts | 2 +- .../workspaces/DatabricksWorkspaceFile.ts | 2 +- .../workspaces/DatabricksWorkspaceNotebook.ts | 2 +- 12 files changed, 82 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b72dc3f..d16c9f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ - new connection manager [Databricks Extensions](README.md/#setup-and-configuration-databricks-extension-connection-manager) - derive cluster for [SQL Browser](README.md/#sql-browser) - change cluster using [Cluster Manager](README.md/#cluster-manager) - + - added File System `wsfs:/` to replace `dbws:/` in the future (currently both are still supported) **v1.5.0:** - added support for [Widgets](README.md/#widgets) when running Notebooks diff --git a/README.md b/README.md index 9990edd..db98cad 100644 --- a/README.md +++ b/README.md @@ -220,10 +220,10 @@ For better visualization of tabluar results this extension includes a dependency Notebook Kernels also support other features like [Files in Repo](https://docs.databricks.com/_static/notebooks/files-in-repos.html) to build libraries within your repo, [_sqldf](https://docs.databricks.com/notebooks/notebooks-use.html#explore-sql-cell-results-in-python-notebooks-natively-using-python) to expose results of SQL cells to Python/Pyspark, `%run` to run other notebooks inline with the current notebook and also [dbutils.notebook.run()](https://docs.databricks.com/dev-tools/databricks-utils.html#notebook-utility-dbutilsnotebook). -Whenever a notebook is opened from either the local sync folder or via the [Virtual File System](#file-system-integration) using `dbws:/` URI, the Databricks notebook kernels are the preferred ones and should appear at the top of the list when you select a kernel. +Whenever a notebook is opened from either the local sync folder or via the [Virtual File System](#file-system-integration) using `wsfs:/` URI, the Databricks notebook kernels are the preferred ones and should appear at the top of the list when you select a kernel. ## Execution Modes -We distinguish between Live-execution and Offline-execution. In Live-execution mode, files are opened directly from Databricks by mounting the Databricks Workspace into your VSCode Workspace using `dbws:/` URI scheme. In this mode there is no intermediate local copy but you work directly against the Databricks Workspace. Everything you run must already exist online in the Databricks Workspace. +We distinguish between Live-execution and Offline-execution. In Live-execution mode, files are opened directly from Databricks by mounting the Databricks Workspace into your VSCode Workspace using `wsfs:/` URI scheme. In this mode there is no intermediate local copy but you work directly against the Databricks Workspace. Everything you run must already exist online in the Databricks Workspace. This is slightly different in Offline-execution where all files you want to work with need to be synced locally first using the [Workspace Manager](#workspace-manager). This is especially important when it comes `%run` which behaves slightly differntly compared to Live-execution mode. `%run` in Offline-execution runs the code from your local file instead of the code that exists in Dtabricks online! Other commands like `dbutils.notebook.run()` always use the code thats currently online so if you have changed the refernced notebook locally, you have to upload it first. This is simply because we cannot easily replicate the behavior of `dbutils.notebook.run()` locally! @@ -250,7 +250,8 @@ You want to upload a local notebook to the Databricks workspace? Simply drag&dro You want to download a file from DBFS? Simply drag&drop it! There are two virtual file systems that come with this extension: -- `dbws:/` to access your notebook from the DAtabricks workspace +- `wsfs:/` to access your notebook from the DAtabricks workspace +- `dbws:/` (LEGACY) - to be replaced by `wsfs:/` in the long term - `dbfs:/` to access files on the Databricks File System (DBFS) - similar to the [DBFS Browser](#dbfs-browser) # SQL Browser diff --git a/package.json b/package.json index 6ecc500..50445cb 100644 --- a/package.json +++ b/package.json @@ -51,12 +51,13 @@ ], "activationEvents": [ "onFileSystem:dbfs", - "onFileSystem:dbws" + "onFileSystem:dbws", + "onFileSystem:wsfs" ], "capabilities": { "virtualWorkspaces": { "supported": "limited", - "description": "In virtual workspaces, Workspace and DBFS browsers are not supported. Please use `dbfs:/` and `dbws:/` mount points instead." + "description": "In virtual workspaces, Workspace and DBFS browsers are not supported. Please use `dbfs:/` and `wsfs:/` mount points instead." }, "untrustedWorkspaces": { "supported": "limited", diff --git a/src/ThisExtension.ts b/src/ThisExtension.ts index a2a98c1..469e965 100644 --- a/src/ThisExtension.ts +++ b/src/ThisExtension.ts @@ -13,6 +13,11 @@ import { DatabricksConnectionManagerDatabricks } from './vscode/treeviews/connec // https://vshaxe.github.io/vscode-extern/vscode/TreeDataProvider.html export abstract class ThisExtension { + static readonly DBFS_SCHEME = "dbfs"; + static readonly WORKSPACE_SCHEME = "wsfs"; + static readonly WORKSPACE_SCHEME_LEGACY = "dbws"; + + private static _context: vscode.ExtensionContext; private static _extension: vscode.Extension; private static _statusBar: vscode.StatusBarItem; diff --git a/src/extension.ts b/src/extension.ts index 54f7d2f..f522e41 100644 --- a/src/extension.ts +++ b/src/extension.ts @@ -100,11 +100,11 @@ export async function activate(context: vscode.ExtensionContext) { // Workspace File System Provider const workspaceProvider = new DatabricksWorkspaceProvider(context); vscode.commands.registerCommand('databricksWorkspace.addToWorkspace', (showMessage: boolean) => { - FSHelper.addToWorkspace(vscode.Uri.parse('dbws:/'), "Databricks - Workspace", showMessage); + FSHelper.addToWorkspace(vscode.Uri.parse(ThisExtension.WORKSPACE_SCHEME +':/'), "Databricks - Workspace", showMessage); }); if (ThisExtension.isInBrowser) { - // in a virtual workspace we always want to add the DBWS mount to the workspace + // in a virtual workspace we always want to add the WSFS/DBWS mount to the workspace vscode.commands.executeCommand('databricksWorkspace.addToWorkspace', false); } @@ -160,11 +160,11 @@ export async function activate(context: vscode.ExtensionContext) { // DBFS File System Provider const dbfsProvider = new DatabricksFileSystemProvider(context); vscode.commands.registerCommand('databricksFS.addToWorkspace', (showMessage: boolean) => { - FSHelper.addToWorkspace(vscode.Uri.parse('dbfs:/'), "Databricks - DBFS", showMessage); + FSHelper.addToWorkspace(vscode.Uri.parse(ThisExtension.DBFS_SCHEME + ':/'), "Databricks - DBFS", showMessage); }); if (ThisExtension.isInBrowser) { - // in a virtual workspace we always want to add the DBWS mount to the workspace + // in a virtual workspace we always want to add the DBFS mount to the workspace vscode.commands.executeCommand('databricksFS.addToWorkspace', false); } diff --git a/src/helpers/Helper.ts b/src/helpers/Helper.ts index 2415a7a..8aff3e6 100644 --- a/src/helpers/Helper.ts +++ b/src/helpers/Helper.ts @@ -73,8 +73,7 @@ export abstract class Helper { try { // three '/' in the beginning indicate a local path // however, there are issues if this.localFilePath also starts with a '/' so we do a replace in this special case - if(typeof(path) == "string") - { + if (typeof (path) == "string") { path = vscode.Uri.parse(("file:///" + path).replace('////', '///')); } await vscode.workspace.fs.stat(path); @@ -100,8 +99,7 @@ export abstract class Helper { static getToken(text: string, separator: string, token: number): string { let parts: string[] = text.split(separator); - if(token < 0) - { + if (token < 0) { return parts.slice(token)[0]; } return parts[token]; @@ -110,8 +108,7 @@ export abstract class Helper { // not working! static runAsyncFunction(func: Function, args: any = undefined): T { return (async function () { - if(args == undefined) - { + if (args == undefined) { return func() } return func(args); @@ -127,7 +124,7 @@ export abstract class Helper { } return immediatelyResolvedPromise as unknown as T; } - + static ensureLocalFolder(path: string, pathIsFile: boolean = false): void { let folder: vscode.Uri = vscode.Uri.file(path); @@ -196,7 +193,7 @@ export abstract class Helper { vscode.commands.executeCommand("vscode.diff", filePath1, filePath2, "Online <-> Local", options); } - + static openLink(link: string): void { vscode.commands.executeCommand('vscode.open', vscode.Uri.parse(link)); @@ -261,13 +258,12 @@ export abstract class Helper { var mDisplay = m > 0 ? `${m.toString().length > 1 ? `${m}` : `0${m}`}` : '00'; var sDisplay = s > 0 ? `${s.toString().length > 1 ? `${s}` : `0${s}`}` : '00'; - return `${hDisplay}:${mDisplay}:${sDisplay}`; + return `${hDisplay}:${mDisplay}:${sDisplay}`; } static getFirstRegexGroup(regexp: RegExp, text: string): string { const array = [...text.matchAll(regexp)]; - if(array.length >= 1) - { + if (array.length >= 1) { return array[0][1]; } return null; @@ -276,4 +272,48 @@ export abstract class Helper { static parseBoolean(value: string): boolean { return value === 'false' || value === 'undefined' || value === 'null' || value === '0' ? false : !!value; } + + static async waitForPromise(promise: Promise, timeout: number): Promise { + // Set a timer that will resolve with null + return new Promise((resolve, reject) => { + const timer = setTimeout(() => resolve(null), timeout); + promise + .then((result) => { + // When the promise resolves, make sure to clear the timer or + // the timer may stick around causing tests to wait + clearTimeout(timer); + resolve(result); + }) + .catch((e) => { + clearTimeout(timer); + reject(e); + }); + }); + } + + static async waitForCondition( + condition: () => Promise, + timeout: number, + interval: number + ): Promise { + // Set a timer that will resolve with null + return new Promise((resolve) => { + let finish: (result: boolean) => void; + const timer = setTimeout(() => finish(false), timeout); + const intervalId = setInterval(() => { + condition() + .then((r) => { + if (r) { + finish(true); + } + }) + .catch((_e) => finish(false)); + }, interval); + finish = (result: boolean) => { + clearTimeout(timer); + clearInterval(intervalId); + resolve(result); + }; + }); + } } \ No newline at end of file diff --git a/src/vscode/filesystemProvider/DatabricksFileSystemProvider.ts b/src/vscode/filesystemProvider/DatabricksFileSystemProvider.ts index 55d5532..f783db1 100644 --- a/src/vscode/filesystemProvider/DatabricksFileSystemProvider.ts +++ b/src/vscode/filesystemProvider/DatabricksFileSystemProvider.ts @@ -8,12 +8,13 @@ import { DatabricksApiService } from '../../databricksApi/databricksApiService'; import { iDatabricksApiDbfsReadResponse } from '../../databricksApi/_types'; import { FSHelper } from '../../helpers/FSHelper'; import { Helper } from '../../helpers/Helper'; +import { ThisExtension } from '../../ThisExtension'; import { iDatabricksFSItem } from '../treeviews/dbfs/iDatabricksFSItem'; export class DatabricksFileSystemProvider implements vscode.FileSystemProvider { constructor(context: vscode.ExtensionContext) { - context.subscriptions.push(vscode.workspace.registerFileSystemProvider('dbfs', this, { isCaseSensitive: true })); + context.subscriptions.push(vscode.workspace.registerFileSystemProvider(ThisExtension.DBFS_SCHEME, this, { isCaseSensitive: true })); } // --- manage file metadata diff --git a/src/vscode/filesystemProvider/DatabricksWorkspaceProvider.ts b/src/vscode/filesystemProvider/DatabricksWorkspaceProvider.ts index 446a085..ffb0996 100644 --- a/src/vscode/filesystemProvider/DatabricksWorkspaceProvider.ts +++ b/src/vscode/filesystemProvider/DatabricksWorkspaceProvider.ts @@ -167,6 +167,8 @@ export class DatabricksWorkspaceProviderItem implements vscode.FileStat, iDatabr export class DatabricksWorkspaceProvider implements vscode.FileSystemProvider { constructor(context: vscode.ExtensionContext) { + context.subscriptions.push(vscode.workspace.registerFileSystemProvider('wsfs', this, { isCaseSensitive: true })); + // legacy, to be removed in the future context.subscriptions.push(vscode.workspace.registerFileSystemProvider('dbws', this, { isCaseSensitive: true })); } diff --git a/src/vscode/notebook/DatabricksKernel.ts b/src/vscode/notebook/DatabricksKernel.ts index 13a5fba..1ea184d 100644 --- a/src/vscode/notebook/DatabricksKernel.ts +++ b/src/vscode/notebook/DatabricksKernel.ts @@ -73,8 +73,8 @@ export class DatabricksKernel implements vscode.NotebookController { } async _onDidOpenNotebookDocument(notebook: vscode.NotebookDocument) { - // set this controller as recommended Kernel for notebooks opened via dbws:/ file system or from or local sync folder - if (notebook.uri.scheme == "dbws" || notebook.uri.toString().startsWith(ThisExtension.ActiveConnection.localSyncFolder.toString())) { + // set this controller as recommended Kernel for notebooks opened via dbws:/, wsfs:/ file system or from or local sync folder + if (notebook.uri.scheme == "dbws" || notebook.uri.scheme == "wsfs" || notebook.uri.toString().startsWith(ThisExtension.ActiveConnection.localSyncFolder.toString())) { this.Controller.updateNotebookAffinity(notebook, vscode.NotebookControllerAffinity.Preferred); } @@ -387,9 +387,10 @@ export class DatabricksKernel implements vscode.NotebookController { } else { // absolute path provided switch (cell.notebook.uri.scheme) { - case "dbws": + case "dbws": // legacy + case "wsfs": runUri = vscode.Uri.file(runFile); - runUri = runUri.with({ scheme: "dbws" }) + runUri = runUri.with({ scheme: "wsfs" }) break; case "file": runUri = await FSHelper.joinPath(ThisExtension.ActiveConnection.localSyncFolder, ThisExtension.ActiveConnection.localSyncSubfolders.Workspace, runFile); @@ -402,7 +403,8 @@ export class DatabricksKernel implements vscode.NotebookController { ThisExtension.log("Executing %run for '" + runUri + "' ..."); try { switch (runUri.scheme) { - case "dbws": + case "dbws": // legacy + case "wsfs": if (!await FSHelper.pathExists(runUri)) { throw vscode.FileSystemError.FileNotFound(runUri); } diff --git a/src/vscode/treeviews/dbfs/DatabricksFSTreeItem.ts b/src/vscode/treeviews/dbfs/DatabricksFSTreeItem.ts index 5df0eae..d01d12f 100644 --- a/src/vscode/treeviews/dbfs/DatabricksFSTreeItem.ts +++ b/src/vscode/treeviews/dbfs/DatabricksFSTreeItem.ts @@ -95,7 +95,7 @@ export class DatabricksFSTreeItem extends vscode.TreeItem implements iDatabricks } get dbfsUri(): vscode.Uri { - return vscode.Uri.parse("dbfs:" + this.path); + return vscode.Uri.parse(ThisExtension.DBFS_SCHEME + ":" + this.path); } public static fromInterface(item: iDatabricksFSItem, parent: DatabricksFSDirectory): DatabricksFSTreeItem { diff --git a/src/vscode/treeviews/workspaces/DatabricksWorkspaceFile.ts b/src/vscode/treeviews/workspaces/DatabricksWorkspaceFile.ts index 1b1b297..c7ecf62 100644 --- a/src/vscode/treeviews/workspaces/DatabricksWorkspaceFile.ts +++ b/src/vscode/treeviews/workspaces/DatabricksWorkspaceFile.ts @@ -185,7 +185,7 @@ export class DatabricksWorkspaceFile extends DatabricksWorkspaceTreeItem { } async compare(): Promise { - let onlineFileTempPath: vscode.Uri = vscode.Uri.parse("dbws:" + this.path); + let onlineFileTempPath: vscode.Uri = vscode.Uri.parse(ThisExtension.WORKSPACE_SCHEME + ":" + this.path); Helper.showDiff(onlineFileTempPath, this.localPath); } diff --git a/src/vscode/treeviews/workspaces/DatabricksWorkspaceNotebook.ts b/src/vscode/treeviews/workspaces/DatabricksWorkspaceNotebook.ts index 8c530c2..638d87b 100644 --- a/src/vscode/treeviews/workspaces/DatabricksWorkspaceNotebook.ts +++ b/src/vscode/treeviews/workspaces/DatabricksWorkspaceNotebook.ts @@ -241,7 +241,7 @@ export class DatabricksWorkspaceNotebook extends DatabricksWorkspaceTreeItem { } async compare(): Promise { - let onlineFileTempPath: vscode.Uri = vscode.Uri.parse("dbws:" + this.path); + let onlineFileTempPath: vscode.Uri = vscode.Uri.parse(ThisExtension.WORKSPACE_SCHEME + ":" + this.path); Helper.showDiff(onlineFileTempPath, this.localPath); } From d5605544a0f8bc30f11f41a6c6e407fed0c485d9 Mon Sep 17 00:00:00 2001 From: Gerhard Brueckl Date: Wed, 22 Feb 2023 14:54:59 +0100 Subject: [PATCH 4/9] add wsfs as new filesystem handle acces token changes --- package.json | 4 +- src/ThisExtension.ts | 43 ++++++++++++------- src/databricksApi/databricksApiService.ts | 14 +++--- .../DatabricksWorkspaceProvider.ts | 5 ++- src/vscode/notebook/DatabricksKernel.ts | 12 +++--- .../DatabricksConnectionManagerAzure.ts | 4 ++ src/vscode/treeviews/dbfs/DatabricksFSFile.ts | 2 +- 7 files changed, 52 insertions(+), 32 deletions(-) diff --git a/package.json b/package.json index 50445cb..865e97f 100644 --- a/package.json +++ b/package.json @@ -437,7 +437,7 @@ "id": "databricksWorkspace", "name": "Workspace", "type": "tree", - "when": "!virtualWorkspace" + "when": "!virtualWorkspace && !paiqo.databricks.isInBrowser" }, { "id": "databricksClusters", @@ -453,7 +453,7 @@ "id": "databricksFS", "name": "DBFS", "type": "tree", - "when": "!virtualWorkspace" + "when": "!virtualWorkspace && !paiqo.databricks.isInBrowser" }, { "id": "databricksSecrets", diff --git a/src/ThisExtension.ts b/src/ThisExtension.ts index 469e965..de7a8e1 100644 --- a/src/ThisExtension.ts +++ b/src/ThisExtension.ts @@ -75,12 +75,11 @@ export abstract class ThisExtension { // handle default if (conManager == "Default") { ThisExtension.log("Default connection manager selected. Trying to find the best connection manager ...") - if (vscode.extensions.getExtension("databricks.databricks123")) { + if (vscode.extensions.getExtension("databricks.databricks")) { ThisExtension.log("Databricks Extension found. Using it as connection manager ..."); conManager = "Databricks Extension"; } - else - { + else { ThisExtension.log("Databricks Extension not found. Using VSCode Settings as connection manager ..."); conManager = "VSCode Settings"; // should open workspace settings with a filter but the filter is not yet working @@ -101,12 +100,6 @@ export abstract class ThisExtension { break; case "Databricks Extension": this._connectionManager = new DatabricksConnectionManagerDatabricks(); - // we hide the Connections Tab as we load all information from the Databricks Extension - vscode.commands.executeCommand( - "setContext", - "paiqo.databricks.hideConnectionManager", - true - ); break; case "Manual Input": this._connectionManager = new DatabricksConnectionManagerManualInput(); @@ -119,14 +112,10 @@ export abstract class ThisExtension { this._connectionManagerText = connectionManager.value as ConnectionManager; } - vscode.commands.executeCommand( - "setContext", - "paiqo.databricks.connectionManager", - this._connectionManagerText - ); - await this.ConnectionManager.initialize(); + await this.setContext(); + return true; } catch (error) { return false; @@ -137,6 +126,27 @@ export abstract class ThisExtension { } + private static async setContext(): Promise { + // we hide the Connections Tab as we load all information from the Databricks Extension + await vscode.commands.executeCommand( + "setContext", + "paiqo.databricks.hideConnectionManager", + ThisExtension._connectionManagerText == "Databricks Extension" + ); + + await vscode.commands.executeCommand( + "setContext", + "paiqo.databricks.connectionManager", + this._connectionManagerText + ); + + await vscode.commands.executeCommand( + "setContext", + "paiqo.databricks.isInBrowser", + this.isInBrowser + ); + } + private static readGlobalSettings(): void { let settingScope: ConfigSettingSource = "Workspace"; @@ -300,7 +310,8 @@ export abstract class ThisExtension { } static get isInBrowser(): boolean { - return process.hasOwnProperty("browser") && process["browser"]; + return vscode.env.uiKind === vscode.UIKind.Web; + //return process.hasOwnProperty("browser") && process["browser"]; } static async updateConfigurationSetting(setting: string, value: any, target: ConfigSettingSource = this._settingScope): Promise { diff --git a/src/databricksApi/databricksApiService.ts b/src/databricksApi/databricksApiService.ts index fab248f..f3ba897 100644 --- a/src/databricksApi/databricksApiService.ts +++ b/src/databricksApi/databricksApiService.ts @@ -40,11 +40,7 @@ export abstract class DatabricksApiService { let headers = await ThisExtension.ConnectionManager.getAuthorizationHeaders(con); this._apiBaseUrl = Helper.trimChar(con.apiRootUrl.with({ path: '', query: '', fragment: '' }).toString(true), '/') + this.API_SUB_URL; - this._headers = { - ...headers, - "Content-Type": 'application/json', - "Accept": 'application/json' - } + this.updateHeaders(headers); ThisExtension.log(`Testing new Databricks API (${con.apiRootUrl}) settings ...`); this._connectionTestRunning = true; @@ -71,6 +67,14 @@ export abstract class DatabricksApiService { public static get isInitialized(): boolean { return DatabricksApiService._isInitialized; } + + public static updateHeaders(authorizationHeaders: any): void { + this._headers = { + ...authorizationHeaders, + "Content-Type": 'application/json', + "Accept": 'application/json' + } + } //#endregion //#region Helpers diff --git a/src/vscode/filesystemProvider/DatabricksWorkspaceProvider.ts b/src/vscode/filesystemProvider/DatabricksWorkspaceProvider.ts index ffb0996..172f535 100644 --- a/src/vscode/filesystemProvider/DatabricksWorkspaceProvider.ts +++ b/src/vscode/filesystemProvider/DatabricksWorkspaceProvider.ts @@ -7,6 +7,7 @@ import { DatabricksApiService } from '../../databricksApi/databricksApiService'; import { iDatabricksWorkspaceItem } from '../treeviews/workspaces/iDatabricksworkspaceItem'; import { Helper } from '../../helpers/Helper'; +import { ThisExtension } from '../../ThisExtension'; export class DatabricksWorkspaceProviderItem implements vscode.FileStat, iDatabricksWorkspaceItem { // vscode.FileStat properties, basically all are read-only @@ -167,9 +168,9 @@ export class DatabricksWorkspaceProviderItem implements vscode.FileStat, iDatabr export class DatabricksWorkspaceProvider implements vscode.FileSystemProvider { constructor(context: vscode.ExtensionContext) { - context.subscriptions.push(vscode.workspace.registerFileSystemProvider('wsfs', this, { isCaseSensitive: true })); + context.subscriptions.push(vscode.workspace.registerFileSystemProvider(ThisExtension.WORKSPACE_SCHEME, this, { isCaseSensitive: true })); // legacy, to be removed in the future - context.subscriptions.push(vscode.workspace.registerFileSystemProvider('dbws', this, { isCaseSensitive: true })); + context.subscriptions.push(vscode.workspace.registerFileSystemProvider(ThisExtension.WORKSPACE_SCHEME_LEGACY, this, { isCaseSensitive: true })); } // --- manage file metadata diff --git a/src/vscode/notebook/DatabricksKernel.ts b/src/vscode/notebook/DatabricksKernel.ts index 1ea184d..059c892 100644 --- a/src/vscode/notebook/DatabricksKernel.ts +++ b/src/vscode/notebook/DatabricksKernel.ts @@ -74,7 +74,7 @@ export class DatabricksKernel implements vscode.NotebookController { async _onDidOpenNotebookDocument(notebook: vscode.NotebookDocument) { // set this controller as recommended Kernel for notebooks opened via dbws:/, wsfs:/ file system or from or local sync folder - if (notebook.uri.scheme == "dbws" || notebook.uri.scheme == "wsfs" || notebook.uri.toString().startsWith(ThisExtension.ActiveConnection.localSyncFolder.toString())) { + if (notebook.uri.scheme == ThisExtension.WORKSPACE_SCHEME_LEGACY || notebook.uri.scheme == ThisExtension.WORKSPACE_SCHEME || notebook.uri.toString().startsWith(ThisExtension.ActiveConnection.localSyncFolder.toString())) { this.Controller.updateNotebookAffinity(notebook, vscode.NotebookControllerAffinity.Preferred); } @@ -387,10 +387,10 @@ export class DatabricksKernel implements vscode.NotebookController { } else { // absolute path provided switch (cell.notebook.uri.scheme) { - case "dbws": // legacy - case "wsfs": + case ThisExtension.WORKSPACE_SCHEME_LEGACY: // legacy + case ThisExtension.WORKSPACE_SCHEME: runUri = vscode.Uri.file(runFile); - runUri = runUri.with({ scheme: "wsfs" }) + runUri = runUri.with({ scheme: ThisExtension.WORKSPACE_SCHEME }) break; case "file": runUri = await FSHelper.joinPath(ThisExtension.ActiveConnection.localSyncFolder, ThisExtension.ActiveConnection.localSyncSubfolders.Workspace, runFile); @@ -403,8 +403,8 @@ export class DatabricksKernel implements vscode.NotebookController { ThisExtension.log("Executing %run for '" + runUri + "' ..."); try { switch (runUri.scheme) { - case "dbws": // legacy - case "wsfs": + case ThisExtension.WORKSPACE_SCHEME_LEGACY: // legacy + case ThisExtension.WORKSPACE_SCHEME: if (!await FSHelper.pathExists(runUri)) { throw vscode.FileSystemError.FileNotFound(runUri); } diff --git a/src/vscode/treeviews/connections/DatabricksConnectionManagerAzure.ts b/src/vscode/treeviews/connections/DatabricksConnectionManagerAzure.ts index 0ea533a..3d8974f 100644 --- a/src/vscode/treeviews/connections/DatabricksConnectionManagerAzure.ts +++ b/src/vscode/treeviews/connections/DatabricksConnectionManagerAzure.ts @@ -7,6 +7,7 @@ import { DatabricksConnectionTreeItem } from './DatabricksConnectionTreeItem'; import { fetch, getProxyAgent, RequestInit, Response } from '@env/fetch'; import { AzureResourceListRepsonse, AzureResource, AzureSubscriptionListRepsonse } from './_types'; import { FSHelper } from '../../../helpers/FSHelper'; +import { DatabricksApiService } from '../../../databricksApi/databricksApiService'; interface AzureConfig { @@ -87,6 +88,9 @@ export class DatabricksConnectionManagerAzure extends DatabricksConnectionManage private static async _onDidChangeSessions(event: vscode.AuthenticationSessionsChangeEvent) { vscode.window.showWarningMessage("Session Changed! " + event.provider.id); + + let headers = await ThisExtension.ConnectionManager.getAuthorizationHeaders(ThisExtension.ActiveConnection); + DatabricksApiService.updateHeaders(headers); } private async getAADAccessToken(scopes: string[], tenantId?: string): Promise { diff --git a/src/vscode/treeviews/dbfs/DatabricksFSFile.ts b/src/vscode/treeviews/dbfs/DatabricksFSFile.ts index a888034..3dece20 100644 --- a/src/vscode/treeviews/dbfs/DatabricksFSFile.ts +++ b/src/vscode/treeviews/dbfs/DatabricksFSFile.ts @@ -222,6 +222,6 @@ export class DatabricksFSFile extends DatabricksFSTreeItem { } async compare(): Promise { - Helper.showDiff(vscode.Uri.parse("dbfs:/" + this.path), this.localPath); + Helper.showDiff(vscode.Uri.parse(ThisExtension.DBFS_SCHEME + ":/" + this.path), this.localPath); } } \ No newline at end of file From 0adc0c8fe8d031326e5a2b54136344f9f0a73c81 Mon Sep 17 00:00:00 2001 From: Gerhard Brueckl Date: Tue, 14 Mar 2023 23:15:38 +0100 Subject: [PATCH 5/9] prepare for release --- README.md | 16 ++-- package.json | 86 ++++++++++++++++--- resources/databricks_sidebar_orig.svg | 1 - src/ThisExtension.ts | 2 +- src/vscode/notebook/DatabricksKernel.ts | 10 ++- .../notebook/DatabricksKernelManager.ts | 2 +- .../treeviews/clusters/iDatabricksCluster.ts | 2 + .../DatabricksConnectionManagerDatabricks.ts | 8 +- src/vscode/treeviews/connections/_types.ts | 2 +- 9 files changed, 101 insertions(+), 28 deletions(-) delete mode 100644 resources/databricks_sidebar_orig.svg diff --git a/README.md b/README.md index db98cad..16f3673 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# VSCode Extension for Databricks +# Databricks Power Tools for VSCode [![Version](https://vsmarketplacebadges.dev/version/paiqo.databricks-vscode.svg?color=blue&style=?style=for-the-badge&logo=visual-studio-code)](https://marketplace.visualstudio.com/items?itemName=paiqo.databricks-vscode) [![Installs](https://vsmarketplacebadges.dev/installs/paiqo.databricks-vscode.svg?color=yellow)](https://marketplace.visualstudio.com/items?itemName=paiqo.databricks-vscode) [![Downloads](https://vsmarketplacebadges.dev/downloads/paiqo.databricks-vscode.svg?color=yellow)](https://marketplace.visualstudio.com/items?itemName=paiqo.databricks-vscode) @@ -6,7 +6,7 @@ ![Databricks-VSCode](/images/Databricks-VSCode.jpg?raw=true "Databricks-VSCode") -This is a Visual Studio Code extension that allows you to work with Databricks locally from VSCode in an efficient way, having everything you need integrated into VS Code - see [Features](#features). It allows you to manage and execute your notebooks, start/stop clusters, execute jobs and much more! +This is a Visual Studio Code extension that allows you to work with Databricks locally from VSCode in an efficient way, having everything you need integrated into VS Code - see [Features](#features). It allows you to execute your notebooks, start/stop clusters, execute jobs and much more! The extensions can be downloaded from the official Visual Studio Code extension gallery: [Databricks VSCode](https://marketplace.visualstudio.com/items?itemName=paiqo.databricks-vscode) @@ -36,6 +36,7 @@ The extensions can be downloaded from the official Visual Studio Code extension - control how notebooks are downloaded (Jupyter notebook, source code, ...) - various other settings - Load Databricks directly from your Azure Account + - Leverage connections configured by the official Databricks VSCode extension - [SQL / Data Browser](#sql-browser) - Browse availabl SQL objects from the Databricks metastore - databases, tables and views, columns, ... @@ -65,11 +66,14 @@ Alternatively it can also be downloaded the `.vsix` directly from the VS Code m Preview-Versions might also be available via github [Releases](https://github.com/paiqo/Databricks-VSCode/releases) from this repository. -# Setup and Configuration (VSCode Connection Manager) +# Setup and Configuration The configuration happens directly via VS Code by simply [opening the settings](https://code.visualstudio.com/docs/getstarted/settings#_creating-user-and-workspace-settings) Then either search for "Databricks" or expand Extensions -> Databricks. -The settings themselves are very well described and it should be easy for you to populate them. Also, not all of them are mandatory! Some of the optional settings are experimental or still work in progress. -To configure multiple Databricks Connections/workspaces, you need to use the JSON editor and add them to `databricks.connections`: +The most important setting to start with is definitly `databricks.connectionManager` as it defines how you manage your connections. There are a couple of differnt options which are described further down below. +All the settings themselves are very well described and it should be easy for you to populate them. Also, not all of them are mandatory, and depend a lot on the connection manager that you have chosen.Some of the optional settings are experimental or still work in progress. + +# Setup and Configuration (VSCode Connection Manager) +Using `VSCode Settings` as your connection manager allows you to define and manage your connections directly from within VSCode via regular VSCode settings. It is recommended to use workspace settings over user settings here as it might get confusing otherwise. The default connection can be configured directly via the settings UI using the `databricks.connection.default.*` settings. To configure multiple Databricks Connections/workspaces, you need to use the JSON editor and add them to `databricks.connections`: ``` json ... @@ -171,7 +175,7 @@ They are documented via VSCode settings documentation. # Setup and Configuration (Databricks Extension Connection Manager) This connection manager leverages the [official Databricks extensions](https://marketplace.visualstudio.com/items?itemName=databricks.databricks) to establish a connection with your Databricks workspace. It only supports a single connection hence the actual Connection Manager tab will be hidden for this connection manager. -It also derives the cluster automatically from the Databricks extensions to source the [SQL Browser](#sql-browser) but also allows you to change it directly from the [Cluster Manager](#cluster-manager) using the `Attach cluster` command. +It also derives the cluster automatically from the Databricks extensions to source the [SQL Browser](#sql-browser) but also allows you to change it directly from the [Cluster Manager](#cluster-manager) using the `Attach cluster` command from the context menu! # Connection Manager ![Connection Manager](/images/ConnectionManager.jpg?raw=true "Connection Manager") diff --git a/package.json b/package.json index 865e97f..bb049c4 100644 --- a/package.json +++ b/package.json @@ -81,7 +81,7 @@ "Databricks CLI Profiles", "Azure", "Databricks Extension", - "Manual Input" + "Manual" ], "default": "Default", "enumDescriptions": [ @@ -418,14 +418,14 @@ "viewsContainers": { "activitybar": [ { - "id": "databricksBar", - "title": "Databricks", - "icon": "resources/databricks_sidebar_orig.svg" + "id": "databricksPowerTools", + "title": "Databricks Power Tools", + "icon": "resources/databricks_sidebar.png" } ] }, "views": { - "databricksBar": [ + "databricksPowerTools": [ { "id": "databricksConnections", "name": "Connections", @@ -482,23 +482,27 @@ { "command": "databricksConnections.refresh", "title": "Refresh", + "category": "Databricks Power Tools", "icon": "$(refresh)" }, { "command": "databricksConnections.add", "title": "Add", + "category": "Databricks Power Tools", "icon": "$(add)", "enablement": "virtualWorkspace" }, { "command": "databricksConnections.settings", "title": "Settings", + "category": "Databricks Power Tools", "icon": "$(gear)", "enablement": "!virtualWorkspace" }, { "command": "databricksConnectionItem.activate", "title": "Activate", + "category": "Databricks Power Tools", "icon": { "light": "resources/light/connected.png", "dark": "resources/dark/connected.png" @@ -507,72 +511,87 @@ { "command": "databricksWorkspace.refresh", "title": "Refresh", + "category": "Databricks Power Tools", "icon": "$(refresh)" }, { "command": "databricksWorkspace.download", "title": "Download", + "category": "Databricks Power Tools", "icon": "$(cloud-download)" }, { "command": "databricksWorkspace.upload", "title": "Upload", + "category": "Databricks Power Tools", "icon": "$(cloud-upload)" }, { "command": "databricksWorkspace.addToWorkspace", "title": "Add Databricks Workspace to VSCode workspace", + "category": "Databricks Power Tools", "icon": "$(symbol-class)" }, { "command": "databricksWorkspaceItem.click", - "title": "Open File" + "title": "Open File", + "category": "Databricks Power Tools" }, { "command": "databricksWorkspaceItem.download", "title": "Download", + "category": "Databricks Power Tools", "icon": "$(cloud-download)" }, { "command": "databricksWorkspaceItem.upload", "title": "Upload", + "category": "Databricks Power Tools", "icon": "$(cloud-upload)" }, { "command": "databricksWorkspaceItem.compare", "title": "Compare", + "category": "Databricks Power Tools", "icon": "$(compare-changes)" }, { "command": "databricksWorkspaceItem.delete", "title": "Delete", + "category": "Databricks Power Tools", "icon": "$(remove-close)" }, { "command": "databricksWorkspaceItem.copyPath", - "title": "Copy Path" + "title": "Copy Path", + "category": "Databricks Power Tools" }, { "command": "databricksWorkspaceItem.openExplorer", - "title": "Open Explorer" + "title": "Open Explorer", + "category": "Databricks Power Tools" }, { "command": "databricksClusters.refresh", "title": "Refresh", + "category": "Databricks Power Tools", "icon": "$(refresh)" }, { "command": "databricksClusters.add", "title": "Add", + "category": "Databricks Power Tools", "icon": "$(add)" }, { "command": "databricksClusterItem.click", - "title": "Open File" + "title": "Open File", + "category": "Databricks Power Tools" }, { "command": "databricksClusterItem.start", "title": "Start", + "category": "Databricks Power Tools", "icon": { "light": "resources/light/start.png", "dark": "resources/dark/start.png" @@ -581,6 +600,7 @@ { "command": "databricksClusterItem.stop", "title": "Stop", + "category": "Databricks Power Tools", "icon": { "light": "resources/light/stop.png", "dark": "resources/dark/stop.png" @@ -589,26 +609,31 @@ { "command": "databricksClusterItem.showDefinition", "title": "JSON Definition", + "category": "Databricks Power Tools", "icon": "$(json)" }, { "command": "databricksClusterItem.delete", "title": "Delete", + "category": "Databricks Power Tools", "icon": "$(remove-close)" }, { "command": "databricksClusterItem.pin", "title": "Pin", + "category": "Databricks Power Tools", "icon": "$(pin)" }, { "command": "databricksClusterItem.unpin", "title": "Unpin", + "category": "Databricks Power Tools", "icon": "$(pinned)" }, { "command": "databricksClusterItem.useForSQL", "title": "Use for SQL Browser", + "category": "Databricks Power Tools", "icon": { "light": "resources/light/stop.png", "dark": "resources/dark/stop.png" @@ -618,36 +643,43 @@ { "command": "databricksClusterItem.attachCluster", "title": "Attach cluster", + "category": "Databricks Power Tools", "icon": "$(plug)", "enablement": "paiqo.databricks.connectionManager == 'Databricks Extension'" }, { "command": "databricksClusterItem.createKernel", "title": "Create Databricks Cluster Kernel", - "shortTitle": "Create Databricks Kernel" + "shortTitle": "Create Databricks Kernel", + "category": "Databricks Power Tools" }, { "command": "databricksClusterItem.restartKernel", "title": "Restart Databricks Cluster Kernel", - "shortTitle": "Restart Databricks Kernel" + "shortTitle": "Restart Databricks Kernel", + "category": "Databricks Power Tools" }, { "command": "databricksJobs.refresh", "title": "Refresh", + "category": "Databricks Power Tools", "icon": "$(refresh)" }, { "command": "databricksJobItem.showDefinition", "title": "JSON Definition", + "category": "Databricks Power Tools", "icon": "$(json)" }, { "command": "databricksJobItem.click", - "title": "Open File" + "title": "Open File", + "category": "Databricks Power Tools" }, { "command": "databricksJobItem.start", "title": "Start", + "category": "Databricks Power Tools", "icon": { "light": "resources/light/start.png", "dark": "resources/dark/start.png" @@ -656,6 +688,7 @@ { "command": "databricksJobItem.stop", "title": "Stop", + "category": "Databricks Power Tools", "icon": { "light": "resources/light/stop.png", "dark": "resources/dark/stop.png" @@ -664,124 +697,149 @@ { "command": "databricksJobItem.openBrowser", "title": "Open Browser", + "category": "Databricks Power Tools", "icon": "$(globe)" }, { "command": "databricksFS.addToWorkspace", "title": "Add DBFS to VSCode workspace", + "category": "Databricks Power Tools", "icon": "$(symbol-class)" }, { "command": "databricksFS.refresh", "title": "Refresh", + "category": "Databricks Power Tools", "icon": "$(refresh)" }, { "command": "databricksFS.addFile", "title": "Add", + "category": "Databricks Power Tools", "icon": "$(new-file)" }, { "command": "databricksFS.addDirectory", "title": "Add", + "category": "Databricks Power Tools", "icon": "$(file-directory-create)" }, { "command": "databricksFSItem.download", "title": "Download", + "category": "Databricks Power Tools", "icon": "$(cloud-download)" }, { "command": "databricksFSItem.upload", "title": "Upload", + "category": "Databricks Power Tools", "icon": "$(cloud-upload)" }, { "command": "databricksFSItem.click", - "title": "Preview File" + "title": "Preview File", + "category": "Databricks Power Tools" }, { "command": "databricksFSItem.addFile", "title": "Add", + "category": "Databricks Power Tools", "icon": "$(new-file)" }, { "command": "databricksFSItem.addDirectory", "title": "Add Directory", + "category": "Databricks Power Tools", "icon": "$(file-directory-create)" }, { "command": "databricksFSItem.delete", "title": "Delete", + "category": "Databricks Power Tools", "icon": "$(remove-close)" }, { "command": "databricksFSItem.copyPath", - "title": "Copy Path" + "title": "Copy Path", + "category": "Databricks Power Tools" }, { "command": "databricksSecrets.refresh", "title": "Refresh", + "category": "Databricks Power Tools", "icon": "$(refresh)" }, { "command": "databricksSecrets.addSecretScope", "title": "Add Secret Scope", + "category": "Databricks Power Tools", "icon": "$(add)" }, { "command": "databricksSecretScope.delete", "title": "Delete", + "category": "Databricks Power Tools", "icon": "$(remove-close)" }, { "command": "databricksSecretScope.addSecret", "title": "Add Secret", + "category": "Databricks Power Tools", "icon": "$(add)" }, { "command": "databricksSecret.update", "title": "Update", + "category": "Databricks Power Tools", "icon": "$(pencil)" }, { "command": "databricksSecret.delete", "title": "Delete", + "category": "Databricks Power Tools", "icon": "$(remove-close)" }, { "command": "databricksSecret.insertCode", "title": "Insert", + "category": "Databricks Power Tools", "icon": "$(code)" }, { "command": "databricksSQL.refresh", "title": "Refresh", + "category": "Databricks Power Tools", "icon": "$(refresh)" }, { "command": "databricksSQLTable.showDefinition", "title": "Show Definition", + "category": "Databricks Power Tools", "icon": "$(json)" }, { "command": "databricksRepos.refresh", "title": "Refresh", + "category": "Databricks Power Tools", "icon": "$(refresh)" }, { "command": "databricksRepo.pull", "title": "Pull", + "category": "Databricks Power Tools", "icon": "$(repo-pull)" }, { "command": "databricksRepo.checkOut", "title": "Check-Out", + "category": "Databricks Power Tools", "icon": "$(repo-clone)" }, { "command": "databricksRepo.delete", "title": "Delete", + "category": "Databricks Power Tools", "icon": "$(remove-close)" }, { diff --git a/resources/databricks_sidebar_orig.svg b/resources/databricks_sidebar_orig.svg deleted file mode 100644 index 3fd304f..0000000 --- a/resources/databricks_sidebar_orig.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/src/ThisExtension.ts b/src/ThisExtension.ts index de7a8e1..5b21ca3 100644 --- a/src/ThisExtension.ts +++ b/src/ThisExtension.ts @@ -101,7 +101,7 @@ export abstract class ThisExtension { case "Databricks Extension": this._connectionManager = new DatabricksConnectionManagerDatabricks(); break; - case "Manual Input": + case "Manual": this._connectionManager = new DatabricksConnectionManagerManualInput(); break; default: diff --git a/src/vscode/notebook/DatabricksKernel.ts b/src/vscode/notebook/DatabricksKernel.ts index 059c892..d0b2804 100644 --- a/src/vscode/notebook/DatabricksKernel.ts +++ b/src/vscode/notebook/DatabricksKernel.ts @@ -48,7 +48,7 @@ export class DatabricksKernel implements vscode.NotebookController { this.notebookType = notebookType; this._cluster = cluster; this._language = language; - this.id = DatabricksKernel.getId(cluster.cluster_id, notebookType); + this.id = DatabricksKernel.getId(this.KernelID, notebookType); this.label = DatabricksKernel.getLabel(cluster.cluster_name); this._executionOrder = 0; @@ -106,8 +106,8 @@ export class DatabricksKernel implements vscode.NotebookController { } // #region Cluster-properties - static getId(clusterId: string, kernelType: KernelType) { - return this.baseId + clusterId + "-" + kernelType; + static getId(kernelId: string, kernelType: KernelType) { + return kernelId + "-" + kernelType; } static getLabel(clusterName: string) { @@ -118,6 +118,10 @@ export class DatabricksKernel implements vscode.NotebookController { return this._controller; } + get KernelID(): string { + return this._cluster.kernel_id ?? this._cluster.cluster_id; + } + get ClusterID(): string { return this._cluster.cluster_id; } diff --git a/src/vscode/notebook/DatabricksKernelManager.ts b/src/vscode/notebook/DatabricksKernelManager.ts index f80e899..5e45c78 100644 --- a/src/vscode/notebook/DatabricksKernelManager.ts +++ b/src/vscode/notebook/DatabricksKernelManager.ts @@ -58,7 +58,7 @@ export abstract class DatabricksKernelManager { } static getNotebookKernelName(cluster: iDatabricksCluster): string { - return cluster.cluster_id + DatabricksKernelManager.NotebookKernelSuffix; + return (cluster.kernel_id ?? cluster.cluster_id) + DatabricksKernelManager.NotebookKernelSuffix; } static getNotebookKernel(cluster: iDatabricksCluster): DatabricksKernel { diff --git a/src/vscode/treeviews/clusters/iDatabricksCluster.ts b/src/vscode/treeviews/clusters/iDatabricksCluster.ts index de93675..792a89a 100644 --- a/src/vscode/treeviews/clusters/iDatabricksCluster.ts +++ b/src/vscode/treeviews/clusters/iDatabricksCluster.ts @@ -1,6 +1,8 @@ import { ClusterState, ClusterSource } from './_types'; export interface iDatabricksCluster { + kernel_id?: string; // to overwrite the id of the kernel thats being created + cluster_id: string; cluster_name: string; state: ClusterState; diff --git a/src/vscode/treeviews/connections/DatabricksConnectionManagerDatabricks.ts b/src/vscode/treeviews/connections/DatabricksConnectionManagerDatabricks.ts index 2b16651..48dbec4 100644 --- a/src/vscode/treeviews/connections/DatabricksConnectionManagerDatabricks.ts +++ b/src/vscode/treeviews/connections/DatabricksConnectionManagerDatabricks.ts @@ -3,6 +3,8 @@ import * as vscode from 'vscode'; import { ThisExtension } from '../../../ThisExtension'; import { iDatabricksConnection } from './iDatabricksConnection'; import { DatabricksConnectionManager } from './DatabricksConnectionManager'; +import { DatabricksKernelManager } from '../../notebook/DatabricksKernelManager'; +import { iDatabricksCluster } from '../clusters/iDatabricksCluster'; export class DatabricksConnectionManagerDatabricks extends DatabricksConnectionManager { @@ -33,8 +35,12 @@ export class DatabricksConnectionManagerDatabricks extends DatabricksConnectionM await this.activateConnection(this.LastActiveConnection, true); - ThisExtension.SQLClusterID = this._databricksConnectionManager.cluster.id; + let cluster = this._databricksConnectionManager.cluster.details as iDatabricksCluster; + ThisExtension.SQLClusterID = cluster.cluster_id; + cluster.cluster_name = "Extension (Generic)"; + cluster.kernel_id = "databricks_extension_generic"; + DatabricksKernelManager.createKernels(cluster); } catch (error) { let msg = "Could not activate Connection '" + this._lastActiveConnectionName + "'!"; ThisExtension.log(msg); diff --git a/src/vscode/treeviews/connections/_types.ts b/src/vscode/treeviews/connections/_types.ts index b7788ea..01e9fd6 100644 --- a/src/vscode/treeviews/connections/_types.ts +++ b/src/vscode/treeviews/connections/_types.ts @@ -10,7 +10,7 @@ export type ConnectionManager = | "Databricks CLI Profiles" | "Azure" | "Databricks Extension" -| "Manual Input" +| "Manual" ; export type ConnectionSource = From 93b1360a738c5f2875d936a10c2effbf9edc7913 Mon Sep 17 00:00:00 2001 From: Gerhard Brueckl Date: Wed, 15 Mar 2023 22:54:04 +0100 Subject: [PATCH 6/9] preps for release 2.0 --- CHANGELOG.md | 3 ++- package-lock.json | 18 +++++++++--------- package.json | 5 +++-- src/ThisExtension.ts | 9 +++++---- .../DatabricksConnectionManagerDatabricks.ts | 19 +++++++++++-------- 5 files changed, 30 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d16c9f8..b902a02 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,11 +1,12 @@ # Release Notes -**v1.9.0:** +**v2.0.0:** - added integration with [official Databricks extensions](https://marketplace.visualstudio.com/items?itemName=databricks.databricks) - new connection manager [Databricks Extensions](README.md/#setup-and-configuration-databricks-extension-connection-manager) - derive cluster for [SQL Browser](README.md/#sql-browser) - change cluster using [Cluster Manager](README.md/#cluster-manager) - added File System `wsfs:/` to replace `dbws:/` in the future (currently both are still supported) + **v1.5.0:** - added support for [Widgets](README.md/#widgets) when running Notebooks diff --git a/package-lock.json b/package-lock.json index a45e914..327572a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "databricks-vscode", - "version": "1.4.0", + "version": "1.9.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "databricks-vscode", - "version": "1.4.0", + "version": "1.9.0", "license": "GPL-3.0-or-later", "devDependencies": { "@types/glob": "^7.2.0", @@ -25,7 +25,7 @@ "process": "^0.11.10", "ts-loader": "^9.3.1", "typescript": "^4.7.4", - "webpack": "^5.74.0", + "webpack": "^5.76.0", "webpack-cli": "^4.10.0" }, "engines": { @@ -4936,9 +4936,9 @@ } }, "node_modules/webpack": { - "version": "5.75.0", - "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.75.0.tgz", - "integrity": "sha512-piaIaoVJlqMsPtX/+3KTTO6jfvrSYgauFVdt8cr9LTHKmcq/AMd4mhzsiP7ZF/PGRNPGA8336jldh9l2Kt2ogQ==", + "version": "5.76.1", + "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.76.1.tgz", + "integrity": "sha512-4+YIK4Abzv8172/SGqObnUjaIHjLEuUasz9EwQj/9xmPPkYJy2Mh03Q/lJfSD3YLzbxy5FeTq5Uw0323Oh6SJQ==", "dev": true, "dependencies": { "@types/eslint-scope": "^3.7.3", @@ -8958,9 +8958,9 @@ "dev": true }, "webpack": { - "version": "5.75.0", - "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.75.0.tgz", - "integrity": "sha512-piaIaoVJlqMsPtX/+3KTTO6jfvrSYgauFVdt8cr9LTHKmcq/AMd4mhzsiP7ZF/PGRNPGA8336jldh9l2Kt2ogQ==", + "version": "5.76.1", + "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.76.1.tgz", + "integrity": "sha512-4+YIK4Abzv8172/SGqObnUjaIHjLEuUasz9EwQj/9xmPPkYJy2Mh03Q/lJfSD3YLzbxy5FeTq5Uw0323Oh6SJQ==", "dev": true, "requires": { "@types/eslint-scope": "^3.7.3", diff --git a/package.json b/package.json index bb049c4..6e00334 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "databricks-vscode", "displayName": "Databricks Power Tools", "description": "Run notebooks cell-by-cell, browse and edit your Databricks Workspace, DBFS, Clusters, Jobs, Secrets, Repos and SQL. Supports Azure Databricks, Databricks on AWS and Databricks on GCP.", - "version": "1.9.0", + "version": "2.0.0", "publisher": "paiqo", "icon": "resources/databricks_extension.png", "author": { @@ -34,6 +34,7 @@ "Azure", "Notebooks", "Data Science", + "Machine Learning", "Other" ], "keywords": [ @@ -1185,7 +1186,7 @@ "process": "^0.11.10", "ts-loader": "^9.3.1", "typescript": "^4.7.4", - "webpack": "^5.74.0", + "webpack": "^5.76.0", "webpack-cli": "^4.10.0" } } \ No newline at end of file diff --git a/src/ThisExtension.ts b/src/ThisExtension.ts index 5b21ca3..a5f025e 100644 --- a/src/ThisExtension.ts +++ b/src/ThisExtension.ts @@ -85,6 +85,8 @@ export abstract class ThisExtension { // should open workspace settings with a filter but the filter is not yet working vscode.commands.executeCommand("workbench.action.openWorkspaceSettings", ThisExtension.configuration.id); } + + this._connectionManagerText = conManager; } switch (conManager) { @@ -108,9 +110,8 @@ export abstract class ThisExtension { this.log("'" + connectionManager + "' is not a valid value for config setting 'databricks.connectionManager!"); } - if (!this._connectionManagerText) { - this._connectionManagerText = connectionManager.value as ConnectionManager; - } + + this._connectionManagerText = conManager await this.ConnectionManager.initialize(); @@ -131,7 +132,7 @@ export abstract class ThisExtension { await vscode.commands.executeCommand( "setContext", "paiqo.databricks.hideConnectionManager", - ThisExtension._connectionManagerText == "Databricks Extension" + this._connectionManagerText == "Databricks Extension" ); await vscode.commands.executeCommand( diff --git a/src/vscode/treeviews/connections/DatabricksConnectionManagerDatabricks.ts b/src/vscode/treeviews/connections/DatabricksConnectionManagerDatabricks.ts index 48dbec4..034af8f 100644 --- a/src/vscode/treeviews/connections/DatabricksConnectionManagerDatabricks.ts +++ b/src/vscode/treeviews/connections/DatabricksConnectionManagerDatabricks.ts @@ -9,6 +9,7 @@ import { iDatabricksCluster } from '../clusters/iDatabricksCluster'; export class DatabricksConnectionManagerDatabricks extends DatabricksConnectionManager { private _databricksConnectionManager: any; + private _apiClient: any; constructor() { super(); @@ -66,15 +67,13 @@ export class DatabricksConnectionManagerDatabricks extends DatabricksConnectionM this._databricksConnectionManager = connectionManager;; let workspaceManager = connectionManager.workspaceClient; - let apiClient = workspaceManager.apiClient; - let host = await apiClient.host; - let token = apiClient.config.token; + this._apiClient = workspaceManager.apiClient; + let host = await this._apiClient.host; let localSyncfolder = connectionManager.syncDestinationMapper?.localUri; this._connections.push({ "apiRootUrl": vscode.Uri.parse(host), - "personalAccessToken": token, "displayName": "Databricks Extension", "localSyncFolder": localSyncfolder.uri, "exportFormats": { @@ -85,14 +84,18 @@ export class DatabricksConnectionManagerDatabricks extends DatabricksConnectionM }, "useCodeCells": false, "_source": "DatabricksExtension" - }) - - let i = 0; - + }) } catch (e) { ThisExtension.log(e); } } updateConnection(updatedCon: iDatabricksConnection): void { } + + async getAuthorizationHeaders(con: iDatabricksConnection): Promise { + const headers: Record = {}; + await this._apiClient.config.authenticate(headers); + + return headers + } } \ No newline at end of file From 1d4e4d3a6b82c03de475a07f9edf5f1fd82fa0c2 Mon Sep 17 00:00:00 2001 From: Gerhard Brueckl Date: Thu, 16 Mar 2023 11:16:52 +0100 Subject: [PATCH 7/9] fixes after review from Fabian --- CHANGELOG.md | 3 ++- README.md | 2 ++ src/vscode/treeviews/clusters/DatabricksCluster.ts | 5 ----- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b902a02..6f07f81 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,8 +5,9 @@ - new connection manager [Databricks Extensions](README.md/#setup-and-configuration-databricks-extension-connection-manager) - derive cluster for [SQL Browser](README.md/#sql-browser) - change cluster using [Cluster Manager](README.md/#cluster-manager) + - automatically create a [Notebook Kernel](README.md/#notebook-kernel) for the configured cluster - added File System `wsfs:/` to replace `dbws:/` in the future (currently both are still supported) - + **v1.5.0:** - added support for [Widgets](README.md/#widgets) when running Notebooks diff --git a/README.md b/README.md index 16f3673..a4d3095 100644 --- a/README.md +++ b/README.md @@ -226,6 +226,8 @@ Notebook Kernels also support other features like [Files in Repo](https://docs.d Whenever a notebook is opened from either the local sync folder or via the [Virtual File System](#file-system-integration) using `wsfs:/` URI, the Databricks notebook kernels are the preferred ones and should appear at the top of the list when you select a kernel. +If you are using the [Databricks Extension Connection Manager](#setup-and-configuration-databricks-extension-connection-manager) we will also create a generic notebook kernel for you which used the configured cluster. + ## Execution Modes We distinguish between Live-execution and Offline-execution. In Live-execution mode, files are opened directly from Databricks by mounting the Databricks Workspace into your VSCode Workspace using `wsfs:/` URI scheme. In this mode there is no intermediate local copy but you work directly against the Databricks Workspace. Everything you run must already exist online in the Databricks Workspace. diff --git a/src/vscode/treeviews/clusters/DatabricksCluster.ts b/src/vscode/treeviews/clusters/DatabricksCluster.ts index 996ad79..f0eb9d3 100644 --- a/src/vscode/treeviews/clusters/DatabricksCluster.ts +++ b/src/vscode/treeviews/clusters/DatabricksCluster.ts @@ -116,11 +116,6 @@ export class DatabricksCluster extends DatabricksClusterTreeItem { return FSHelper.joinPathSync(ThisExtension.rootUri, 'resources', theme, state + '.png'); } - readonly command = { - command: 'databricksClusterItem.click', title: "Open File", arguments: [this] - }; - - get definition(): iDatabricksCluster { return this._definition; } From 47f17892e50cddbc520de5cd6ec47806f5f2e9a2 Mon Sep 17 00:00:00 2001 From: Gerhard Brueckl Date: Mon, 20 Mar 2023 12:53:16 +0100 Subject: [PATCH 8/9] update versions --- package-lock.json | 124 +++++++++++++++++++++++++++++++++++----------- package.json | 2 +- 2 files changed, 96 insertions(+), 30 deletions(-) diff --git a/package-lock.json b/package-lock.json index 327572a..761bd15 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "databricks-vscode", - "version": "1.9.0", + "version": "2.0.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "databricks-vscode", - "version": "1.9.0", + "version": "2.0.0", "license": "GPL-3.0-or-later", "devDependencies": { "@types/glob": "^7.2.0", @@ -15,7 +15,7 @@ "@types/rimraf": "^2.0.5", "@types/vscode": "^1.68.0", "@vscode/test-web": "*", - "@vscode/vsce": "^2.15.0", + "@vscode/vsce": "^2.18.0", "eslint": "^8.19.0", "glob": "^8.0.3", "mocha": "^10.0.0", @@ -331,9 +331,9 @@ } }, "node_modules/@vscode/vsce": { - "version": "2.15.0", - "resolved": "https://registry.npmjs.org/@vscode/vsce/-/vsce-2.15.0.tgz", - "integrity": "sha512-c+qS5KSX4jO3RuGqeNQHqci4+WrcmLxHAwiWTR3PDR6wXzV1fQJxybueUOojXcqvsJR3W2AeROrpf+302ZkTfg==", + "version": "2.18.0", + "resolved": "https://registry.npmjs.org/@vscode/vsce/-/vsce-2.18.0.tgz", + "integrity": "sha512-tUA3XoKx5xjoi3EDcngk0VUYMhvfXLhS4s7CntpLPh1qtLYtgSCexTIMUHkCy6MqyozRW98bdW3a2yHPEADRnQ==", "dev": true, "dependencies": { "azure-devops-node-api": "^11.0.1", @@ -342,7 +342,7 @@ "commander": "^6.1.0", "glob": "^7.0.6", "hosted-git-info": "^4.0.2", - "keytar": "^7.7.0", + "jsonc-parser": "^3.2.0", "leven": "^3.1.0", "markdown-it": "^12.3.2", "mime": "^1.3.4", @@ -362,6 +362,9 @@ }, "engines": { "node": ">= 14" + }, + "optionalDependencies": { + "keytar": "^7.7.0" } }, "node_modules/@vscode/vsce/node_modules/ansi-styles": { @@ -1146,7 +1149,8 @@ "version": "1.1.4", "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", - "dev": true + "dev": true, + "optional": true }, "node_modules/chrome-trace-event": { "version": "1.0.3", @@ -1372,6 +1376,7 @@ "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==", "dev": true, + "optional": true, "dependencies": { "mimic-response": "^3.1.0" }, @@ -1483,6 +1488,7 @@ "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", "dev": true, + "optional": true, "engines": { "node": ">=4.0.0" } @@ -1523,6 +1529,7 @@ "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.1.tgz", "integrity": "sha512-463v3ZeIrcWtdgIg6vI6XUncguvr2TnGl4SzDXinkt9mSLpBJKXT3mW6xT3VQdDN11+WVs29pgvivTc4Lp8v+w==", "dev": true, + "optional": true, "engines": { "node": ">=8" } @@ -1887,6 +1894,7 @@ "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==", "dev": true, + "optional": true, "engines": { "node": ">=6" } @@ -2128,7 +2136,8 @@ "version": "0.0.0", "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==", - "dev": true + "dev": true, + "optional": true }, "node_modules/glob": { "version": "8.0.3", @@ -2486,7 +2495,8 @@ "version": "1.3.8", "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", - "dev": true + "dev": true, + "optional": true }, "node_modules/interpret": { "version": "2.2.0", @@ -2718,6 +2728,12 @@ "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==", "dev": true }, + "node_modules/jsonc-parser": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/jsonc-parser/-/jsonc-parser-3.2.0.tgz", + "integrity": "sha512-gfFQZrcTc8CnKXp6Y4/CBT3fTc0OVuDofpre4aEeEpSBPV5X5v4+Vmx+8snU7RLPrNHPKSgLxGo9YuQzz20o+w==", + "dev": true + }, "node_modules/keygrip": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/keygrip/-/keygrip-1.1.0.tgz", @@ -2736,6 +2752,7 @@ "integrity": "sha512-VPD8mtVtm5JNtA2AErl6Chp06JBfy7diFQ7TQQhdpWOl6MrCRB+eRbvAZUsbGQS9kiMq0coJsy0W0vHpDCkWsQ==", "dev": true, "hasInstallScript": true, + "optional": true, "dependencies": { "node-addon-api": "^4.3.0", "prebuild-install": "^7.0.1" @@ -3145,6 +3162,7 @@ "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==", "dev": true, + "optional": true, "engines": { "node": ">=10" }, @@ -3177,7 +3195,8 @@ "version": "0.5.3", "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==", - "dev": true + "dev": true, + "optional": true }, "node_modules/mocha": { "version": "10.1.0", @@ -3364,7 +3383,8 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-1.0.2.tgz", "integrity": "sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg==", - "dev": true + "dev": true, + "optional": true }, "node_modules/native-ext-loader": { "version": "2.3.0", @@ -3398,6 +3418,7 @@ "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.28.0.tgz", "integrity": "sha512-fRlDb4I0eLcQeUvGq7IY3xHrSb0c9ummdvDSYWfT9+LKP+3jCKw/tKoqaM7r1BAoiAC6GtwyjaGnOz6B3OtF+A==", "dev": true, + "optional": true, "dependencies": { "semver": "^7.3.5" }, @@ -3410,6 +3431,7 @@ "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz", "integrity": "sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==", "dev": true, + "optional": true, "dependencies": { "lru-cache": "^6.0.0" }, @@ -3424,7 +3446,8 @@ "version": "4.3.0", "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-4.3.0.tgz", "integrity": "sha512-73sE9+3UaLYYFmDsFZnqCInzPyh3MqIwZO9cw58yIqAZhONrrabrYyYe3TuIqtIiOuTXVhsGau8hcrhhwSsDIQ==", - "dev": true + "dev": true, + "optional": true }, "node_modules/node-domexception": { "version": "1.0.0", @@ -3851,6 +3874,7 @@ "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.1.tgz", "integrity": "sha512-jAXscXWMcCK8GgCoHOfIr0ODh5ai8mj63L2nWrjuAgXE6tDyYGnx4/8o/rCgU+B4JSyZBKbeZqzhtwtC3ovxjw==", "dev": true, + "optional": true, "dependencies": { "detect-libc": "^2.0.0", "expand-template": "^2.0.3", @@ -3901,6 +3925,7 @@ "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", "dev": true, + "optional": true, "dependencies": { "end-of-stream": "^1.1.0", "once": "^1.3.1" @@ -3964,6 +3989,7 @@ "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", "dev": true, + "optional": true, "dependencies": { "deep-extend": "^0.6.0", "ini": "~1.3.0", @@ -3979,6 +4005,7 @@ "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", "dev": true, + "optional": true, "engines": { "node": ">=0.10.0" } @@ -4380,7 +4407,8 @@ "type": "consulting", "url": "https://feross.org/support" } - ] + ], + "optional": true }, "node_modules/simple-get": { "version": "4.0.1", @@ -4401,6 +4429,7 @@ "url": "https://feross.org/support" } ], + "optional": true, "dependencies": { "decompress-response": "^6.0.0", "once": "^1.3.1", @@ -4535,6 +4564,7 @@ "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.1.tgz", "integrity": "sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==", "dev": true, + "optional": true, "dependencies": { "chownr": "^1.1.1", "mkdirp-classic": "^0.5.2", @@ -4547,6 +4577,7 @@ "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", "dev": true, + "optional": true, "dependencies": { "buffer": "^5.5.0", "inherits": "^2.0.4", @@ -4558,6 +4589,7 @@ "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", "dev": true, + "optional": true, "dependencies": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", @@ -4572,6 +4604,7 @@ "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", "dev": true, + "optional": true, "dependencies": { "bl": "^4.0.3", "end-of-stream": "^1.4.1", @@ -4761,6 +4794,7 @@ "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", "integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==", "dev": true, + "optional": true, "dependencies": { "safe-buffer": "^5.0.1" }, @@ -5518,9 +5552,9 @@ } }, "@vscode/vsce": { - "version": "2.15.0", - "resolved": "https://registry.npmjs.org/@vscode/vsce/-/vsce-2.15.0.tgz", - "integrity": "sha512-c+qS5KSX4jO3RuGqeNQHqci4+WrcmLxHAwiWTR3PDR6wXzV1fQJxybueUOojXcqvsJR3W2AeROrpf+302ZkTfg==", + "version": "2.18.0", + "resolved": "https://registry.npmjs.org/@vscode/vsce/-/vsce-2.18.0.tgz", + "integrity": "sha512-tUA3XoKx5xjoi3EDcngk0VUYMhvfXLhS4s7CntpLPh1qtLYtgSCexTIMUHkCy6MqyozRW98bdW3a2yHPEADRnQ==", "dev": true, "requires": { "azure-devops-node-api": "^11.0.1", @@ -5529,6 +5563,7 @@ "commander": "^6.1.0", "glob": "^7.0.6", "hosted-git-info": "^4.0.2", + "jsonc-parser": "^3.2.0", "keytar": "^7.7.0", "leven": "^3.1.0", "markdown-it": "^12.3.2", @@ -6149,7 +6184,8 @@ "version": "1.1.4", "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", - "dev": true + "dev": true, + "optional": true }, "chrome-trace-event": { "version": "1.0.3", @@ -6321,6 +6357,7 @@ "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==", "dev": true, + "optional": true, "requires": { "mimic-response": "^3.1.0" } @@ -6408,7 +6445,8 @@ "version": "0.6.0", "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", - "dev": true + "dev": true, + "optional": true }, "deep-is": { "version": "0.1.4", @@ -6438,7 +6476,8 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.1.tgz", "integrity": "sha512-463v3ZeIrcWtdgIg6vI6XUncguvr2TnGl4SzDXinkt9mSLpBJKXT3mW6xT3VQdDN11+WVs29pgvivTc4Lp8v+w==", - "dev": true + "dev": true, + "optional": true }, "diff": { "version": "5.0.0", @@ -6702,7 +6741,8 @@ "version": "2.0.3", "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==", - "dev": true + "dev": true, + "optional": true }, "fast-deep-equal": { "version": "3.1.3", @@ -6879,7 +6919,8 @@ "version": "0.0.0", "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==", - "dev": true + "dev": true, + "optional": true }, "glob": { "version": "8.0.3", @@ -7136,7 +7177,8 @@ "version": "1.3.8", "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", - "dev": true + "dev": true, + "optional": true }, "interpret": { "version": "2.2.0", @@ -7310,6 +7352,12 @@ "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==", "dev": true }, + "jsonc-parser": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/jsonc-parser/-/jsonc-parser-3.2.0.tgz", + "integrity": "sha512-gfFQZrcTc8CnKXp6Y4/CBT3fTc0OVuDofpre4aEeEpSBPV5X5v4+Vmx+8snU7RLPrNHPKSgLxGo9YuQzz20o+w==", + "dev": true + }, "keygrip": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/keygrip/-/keygrip-1.1.0.tgz", @@ -7324,6 +7372,7 @@ "resolved": "https://registry.npmjs.org/keytar/-/keytar-7.9.0.tgz", "integrity": "sha512-VPD8mtVtm5JNtA2AErl6Chp06JBfy7diFQ7TQQhdpWOl6MrCRB+eRbvAZUsbGQS9kiMq0coJsy0W0vHpDCkWsQ==", "dev": true, + "optional": true, "requires": { "node-addon-api": "^4.3.0", "prebuild-install": "^7.0.1" @@ -7651,7 +7700,8 @@ "version": "3.1.0", "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==", - "dev": true + "dev": true, + "optional": true }, "minimatch": { "version": "3.1.2", @@ -7672,7 +7722,8 @@ "version": "0.5.3", "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==", - "dev": true + "dev": true, + "optional": true }, "mocha": { "version": "10.1.0", @@ -7826,7 +7877,8 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-1.0.2.tgz", "integrity": "sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg==", - "dev": true + "dev": true, + "optional": true }, "native-ext-loader": { "version": "2.3.0", @@ -7857,6 +7909,7 @@ "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.28.0.tgz", "integrity": "sha512-fRlDb4I0eLcQeUvGq7IY3xHrSb0c9ummdvDSYWfT9+LKP+3jCKw/tKoqaM7r1BAoiAC6GtwyjaGnOz6B3OtF+A==", "dev": true, + "optional": true, "requires": { "semver": "^7.3.5" }, @@ -7866,6 +7919,7 @@ "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz", "integrity": "sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==", "dev": true, + "optional": true, "requires": { "lru-cache": "^6.0.0" } @@ -7876,7 +7930,8 @@ "version": "4.3.0", "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-4.3.0.tgz", "integrity": "sha512-73sE9+3UaLYYFmDsFZnqCInzPyh3MqIwZO9cw58yIqAZhONrrabrYyYe3TuIqtIiOuTXVhsGau8hcrhhwSsDIQ==", - "dev": true + "dev": true, + "optional": true }, "node-domexception": { "version": "1.0.0", @@ -8182,6 +8237,7 @@ "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.1.tgz", "integrity": "sha512-jAXscXWMcCK8GgCoHOfIr0ODh5ai8mj63L2nWrjuAgXE6tDyYGnx4/8o/rCgU+B4JSyZBKbeZqzhtwtC3ovxjw==", "dev": true, + "optional": true, "requires": { "detect-libc": "^2.0.0", "expand-template": "^2.0.3", @@ -8220,6 +8276,7 @@ "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", "dev": true, + "optional": true, "requires": { "end-of-stream": "^1.1.0", "once": "^1.3.1" @@ -8260,6 +8317,7 @@ "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", "dev": true, + "optional": true, "requires": { "deep-extend": "^0.6.0", "ini": "~1.3.0", @@ -8271,7 +8329,8 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", - "dev": true + "dev": true, + "optional": true } } }, @@ -8551,13 +8610,15 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz", "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==", - "dev": true + "dev": true, + "optional": true }, "simple-get": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.1.tgz", "integrity": "sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==", "dev": true, + "optional": true, "requires": { "decompress-response": "^6.0.0", "once": "^1.3.1", @@ -8664,6 +8725,7 @@ "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.1.tgz", "integrity": "sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==", "dev": true, + "optional": true, "requires": { "chownr": "^1.1.1", "mkdirp-classic": "^0.5.2", @@ -8676,6 +8738,7 @@ "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", "dev": true, + "optional": true, "requires": { "buffer": "^5.5.0", "inherits": "^2.0.4", @@ -8687,6 +8750,7 @@ "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", "dev": true, + "optional": true, "requires": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", @@ -8698,6 +8762,7 @@ "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", "dev": true, + "optional": true, "requires": { "bl": "^4.0.3", "end-of-stream": "^1.4.1", @@ -8830,6 +8895,7 @@ "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", "integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==", "dev": true, + "optional": true, "requires": { "safe-buffer": "^5.0.1" } diff --git a/package.json b/package.json index 6e00334..440d512 100644 --- a/package.json +++ b/package.json @@ -1175,7 +1175,7 @@ "@types/node": "^16.11.7", "@types/rimraf": "^2.0.5", "@types/vscode": "^1.68.0", - "@vscode/vsce": "^2.15.0", + "@vscode/vsce": "^2.18.0", "@vscode/test-web": "*", "eslint": "^8.19.0", "glob": "^8.0.3", From 8341f99a7daf88ee90b748e52f1f0cc604c2e57e Mon Sep 17 00:00:00 2001 From: Gerhard Brueckl Date: Mon, 20 Mar 2023 12:56:25 +0100 Subject: [PATCH 9/9] added open todos --- .../DatabricksConnectionManagerDatabricks.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/vscode/treeviews/connections/DatabricksConnectionManagerDatabricks.ts b/src/vscode/treeviews/connections/DatabricksConnectionManagerDatabricks.ts index 034af8f..0493e13 100644 --- a/src/vscode/treeviews/connections/DatabricksConnectionManagerDatabricks.ts +++ b/src/vscode/treeviews/connections/DatabricksConnectionManagerDatabricks.ts @@ -15,6 +15,12 @@ export class DatabricksConnectionManagerDatabricks extends DatabricksConnectionM super(); } + // TODO + /* + there is also an event that fires when the Databricks Extension config is changed + in that case we should also re-initialize the connection manager + */ + async initialize(): Promise { ThisExtension.log("Initializing ConnectionManager Databricks Extension ..."); this._initialized = false; @@ -57,6 +63,10 @@ export class DatabricksConnectionManagerDatabricks extends DatabricksConnectionM const databricksExtension: vscode.Extension = vscode.extensions.getExtension("databricks.databricks"); if (!databricksExtension) { vscode.window.showErrorMessage("Please install the Databricks extension ('databricks.databricks') first!"); + // TODO + /* + You can trigger the installation using the workbench.extensions.installExtension command (see https://code.visualstudio.com/api/references/commands). + */ return; }