preview for v2.1.0 with notebooks for scala, r and sql

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Release Notes
 
+**v2.1.0:**
+- introduced a new notebook type for `Scala`, `SQL` and `R`
+- removed interactive notebook kernel as it never really worked
+  - replaced it with new Databricks kernel to work with the new notebook type
+- minor fix to handle `pip install` more generic
+
 **v2.0.3:**
 - fixed bug with `Buffer` introduced in `v2.0.2`
 

package.json

@@ -2,7 +2,7 @@
 	"name": "databricks-vscode",
 	"displayName": "Databricks Power Tools",
 	"description": "Run notebooks cell-by-cell, browse and edit your Databricks Workspace, DBFS, Clusters, Jobs, Secrets, Repos and SQL. Supports Azure Databricks, Databricks on AWS and Databricks on GCP.",
-	"version": "2.0.3",
+	"version": "2.1.0",
 	"publisher": "paiqo",
 	"icon": "resources/databricks_extension.png",
 	"author": {
@@ -1158,7 +1158,25 @@
 					"when": "notebookKernel =~ /^paiqo.databricks-vscode\\// && isWorkspaceTrusted || isWorkspaceTrusted && !notebookKernel"
 				}
 			]
-		}
+		},
+		"notebooks": [
+			{
+				"type": "databricks-notebook",
+				"displayName": "Databricks Notebook",
+				"selector": [
+					{
+						"filenamePattern": "*.sql"
+					},
+					{
+						"filenamePattern": "*.scala"
+					},
+					{
+						"filenamePattern": "*.r"
+					}
+				],
+				"priority": "default"
+			}
+		]
 	},
 	"scripts": {
 		"vscode:prepublish": "npm run package",
@@ -1189,4 +1207,4 @@
 		"webpack-cli": "^4.10.0",
 		"buffer": "^6.0.3"
 	}
-}
+}

src/databricksApi/_types.ts

@@ -41,6 +41,7 @@ export interface iDatabricksApiJobsRunsListResponse {
 
 export interface iDatabricksApiContextsCreateResponse {
 	id: 			string;
+	error?:			string;
 }
 
 export interface iDatabricksApiContextsStatusResponse {
@@ -49,34 +50,40 @@ export interface iDatabricksApiContextsStatusResponse {
 }
 
 export interface iDatabricksApiContextsDestroyResponse {
-	id: 			string;
+	id?: 			string;
+	error?:			string;
 }
 
 export interface iDatabricksApiCommandsExecuteResponse {
-	id: 			string;
+	id?: 			string;
+	error?:			string;
 }
 
 export interface iDatabricksApiCommandsCancelResponse {
-	id: 			string;
+	id?: 			string;
+	error?:			string;
 }
 
 export interface iDatabricksApiCommandsStatusResponse {
-	id: 			string;
-	status:			string;
-	results:		any;
+	id?: 			string;
+	status?:			string;
+	results?:		any;
+	error?:			string;
 }
 
 export class ExecutionContext {
-	context_id: 	string;
-	cluster_id: 	string;
-	language:		string;
+	context_id?: 	string;
+	cluster_id?: 	string;
+	language?:		string;
+	error?:			string;
 }
 
 export class ExecutionCommand {
-	command_id: 	string;
-	context_id: 	string;
-	cluster_id: 	string;
-	language:		string;
+	command_id?: 	string;
+	context_id?: 	string;
+	cluster_id?: 	string;
+	language?:		string;
+	error?:			string;
 }
 
 export interface iDatabricksApiClustersSparkVersionsResponse {

src/databricksApi/databricksApiService.ts

@@ -412,6 +412,10 @@ export abstract class DatabricksApiService {
 
 		let response: iDatabricksApiContextsCreateResponse = await this.post<iDatabricksApiContextsCreateResponse>(endpoint, body);
 
+		if (!response || response.error) {
+			return response;
+		}
+
 		let ret = {
 			"cluster_id": cluster_id,
 			"language": language,
@@ -427,6 +431,10 @@ export abstract class DatabricksApiService {
 
 		let response: iDatabricksApiContextsDestroyResponse = await this.post<iDatabricksApiContextsDestroyResponse>(endpoint, body);
 
+		if (!response || response.error) {
+			return response;
+		}
+
 		return response;
 	}
 
@@ -441,6 +449,10 @@ export abstract class DatabricksApiService {
 
 		let response: iDatabricksApiCommandsExecuteResponse = await this.post<iDatabricksApiCommandsExecuteResponse>(endpoint, body);
 
+		if (!response || response.error) {
+			return response;
+		}
+
 		let ret = {
 			"command_id": response.id,
 			"cluster_id": context.cluster_id,
@@ -461,6 +473,10 @@ export abstract class DatabricksApiService {
 
 		let response: iDatabricksApiCommandsStatusResponse = await this.get<iDatabricksApiCommandsStatusResponse>(endpoint, body);
 
+		if (!response || response.error) {
+			return response;
+		}
+
 		return response;
 	}
 
@@ -486,6 +502,10 @@ export abstract class DatabricksApiService {
 
 		let response: iDatabricksApiCommandsCancelResponse = await this.post<iDatabricksApiCommandsCancelResponse>(endpoint, body);
 
+		if (!response || response.error) {
+			return response;
+		}
+
 		let ret = {
 			"command_id": response.id,
 			"cluster_id": command.cluster_id,

src/extension.ts

@@ -34,6 +34,7 @@ import { FSHelper } from './helpers/FSHelper';
 import { DatabricksKernelManager } from './vscode/notebook/DatabricksKernelManager';
 import { DatabricksSQLTreeItem } from './vscode/treeviews/sql/DatabricksSQLTreeItem';
 import { DatabricksSecretTreeItem } from './vscode/treeviews/secrets/DatabricksSecretTreeItem';
+import { DatabricksNotebookSerializer } from './vscode/notebook/DatabricksNotebookSerializer';
 
 export async function activate(context: vscode.ExtensionContext) {
 
@@ -64,10 +65,12 @@ export async function activate(context: vscode.ExtensionContext) {
 
 	ThisExtension.setStatusBar("Initialized!");
 
+	let notebookSerializer = new DatabricksNotebookSerializer(context);
+
 	ThisExtension.setStatusBar("Initializing Kernels ...", true);
 	DatabricksKernelManager.initialize();
 	vscode.commands.registerCommand('databricksKernel.restart',
-		(notebook: { notebookEditor: { notebookUri: vscode.Uri } } | undefined | vscode.Uri) => DatabricksKernelManager.restartNotebookKernel(notebook)
+		(notebook: { notebookEditor: { notebookUri: vscode.Uri } } | undefined | vscode.Uri) => DatabricksKernelManager.restartJupyterKernel(notebook)
 	);
 	vscode.commands.registerCommand('databricksKernel.updateWidgets',
 		(notebook: { notebookEditor: { notebookUri: vscode.Uri } } | undefined | vscode.Uri) => DatabricksKernelManager.updateWidgets(notebook)

src/vscode/notebook/DatabricksKernel.ts

@@ -25,7 +25,7 @@ export type NotebookMagic =
 
 export type KernelType =
 	"jupyter-notebook"
-	| "interactive"
+	| "databricks-notebook"
 
 // https://code.visualstudio.com/blogs/2021/11/08/custom-notebooks
 export class DatabricksKernel implements vscode.NotebookController {
@@ -96,7 +96,7 @@ export class DatabricksKernel implements vscode.NotebookController {
 		let execContext: ExecutionContext = this.getNotebookContext(notebook.uri);
 		if (!execContext) {
 			ThisExtension.setStatusBar("Initializing Kernel ...", true);
-			execContext = await this.initializeExecutionContext()
+			execContext = await this.initializeExecutionContext(notebook.metadata?.notebookLanguage?.databricksLanguage);
 			this.setNotebookContext(notebook.uri, execContext);
 			await this.updateRepoContext(notebook.uri);
 			ThisExtension.setStatusBar("Kernel initialized!");
@@ -205,8 +205,8 @@ export class DatabricksKernel implements vscode.NotebookController {
 		}
 	}
 
-	private async initializeExecutionContext(): Promise<ExecutionContext> {
-		return await DatabricksApiService.getExecutionContext(this.ClusterID, this.Language);
+	private async initializeExecutionContext(language?: ContextLanguage): Promise<ExecutionContext> {
+		return await DatabricksApiService.getExecutionContext(this.ClusterID, language ?? this.Language);
 	}
 
 	setNotebookContext(notebookUri: vscode.Uri, context: ExecutionContext): void {
@@ -282,13 +282,11 @@ export class DatabricksKernel implements vscode.NotebookController {
 		}
 	}
 
-
-
 	private async parseCommand(cell: vscode.NotebookCell, executionContext: ExecutionContext): Promise<[ContextLanguage, string, NotebookMagic]> {
 		let cmd: string = cell.document.getText();
 		let magicText: string = undefined;
 		let commandText: string = cmd;
-		let language: ContextLanguage = this.Language;
+		let language: ContextLanguage = executionContext.language as ContextLanguage;
 		if (cmd[0] == "%") {
 			let lines = cmd.split('\n');
 			magicText = lines[0].split(" ")[0].slice(1).trim();
@@ -484,6 +482,19 @@ export class DatabricksKernel implements vscode.NotebookController {
 				return;
 			});
 
+			if (command.error) {
+				execution.appendOutput(new vscode.NotebookCellOutput([
+					vscode.NotebookCellOutputItem.text(command.error, 'text/html')
+				]));
+
+				execution.appendOutput(new vscode.NotebookCellOutput([
+					vscode.NotebookCellOutputItem.error(new Error(command.error)),
+				]));
+
+				execution.end(false, Date.now());
+				return;
+			}
+
 			let result = await DatabricksApiService.getCommandResult(command, true);
 
 			if (result.results.resultType == "table") {
@@ -570,7 +581,7 @@ export class DatabricksKernel implements vscode.NotebookController {
 			}
 
 
-			if (["pip"].includes(magic)) {
+			if (["pip"].includes(magic) || commandText.includes("pip install")) {
 				await this.updateRepoContext(cell.notebook.uri);
 			}
 

src/vscode/notebook/DatabricksKernelManager.ts

@@ -7,8 +7,8 @@ import { DatabricksKernel } from './DatabricksKernel';
 
 
 export abstract class DatabricksKernelManager {
-	private static NotebookKernelSuffix: string = "-jupyter-notebook";
-	private static InteractiveKernelSuffix: string = "-interactive";
+	private static JupyterKernelSuffix: string = "-jupyter-notebook";
+	private static DatabricksKernelSuffix: string = "-databricks-notebook";
 
 	private static _kernels: Map<string, DatabricksKernel> = new Map<string, DatabricksKernel>();
 
@@ -57,40 +57,40 @@ export abstract class DatabricksKernelManager {
 		return this._kernels.get(kernelName);
 	}
 
-	static getNotebookKernelName(cluster: iDatabricksCluster): string {
-		return (cluster.kernel_id ?? cluster.cluster_id) + DatabricksKernelManager.NotebookKernelSuffix;
+	static getJupyterKernelName(cluster: iDatabricksCluster): string {
+		return (cluster.kernel_id ?? cluster.cluster_id) + DatabricksKernelManager.JupyterKernelSuffix;
 	}
 
-	static getNotebookKernel(cluster: iDatabricksCluster): DatabricksKernel {
-		return this.getKernel(this.getNotebookKernelName(cluster));
+	static getJupyterKernel(cluster: iDatabricksCluster): DatabricksKernel {
+		return this.getKernel(this.getJupyterKernelName(cluster));
 	}
 
-	static notebookKernelExists(cluster: iDatabricksCluster): boolean {
-		if (this.getKernel(this.getNotebookKernelName(cluster))) {
+	static jupyterKernelExists(cluster: iDatabricksCluster): boolean {
+		if (this.getKernel(this.getJupyterKernelName(cluster))) {
 			return true;
 		}
 		return false;
 	}
 
-	static getInteractiveKernelName(cluster: iDatabricksCluster): string {
-		return cluster.cluster_id + DatabricksKernelManager.InteractiveKernelSuffix
+	static getDatabricksKernelName(cluster: iDatabricksCluster): string {
+		return cluster.cluster_id + DatabricksKernelManager.DatabricksKernelSuffix
 	}
 
-	static getInteractiveKernel(cluster: iDatabricksCluster): DatabricksKernel {
-		return this.getKernel(this.getInteractiveKernelName(cluster));
+	static getDatabricksKernel(cluster: iDatabricksCluster): DatabricksKernel {
+		return this.getKernel(this.getDatabricksKernelName(cluster));
 	}
 
-	static interactiveKernelExists(cluster: iDatabricksCluster): boolean {
-		if (this.getInteractiveKernel(cluster)) {
+	static databricksKernelExists(cluster: iDatabricksCluster): boolean {
+		if (this.getDatabricksKernel(cluster)) {
 			return true;
 		}
 		return false;
 	}
 
 	static async createKernels(cluster: iDatabricksCluster, logMessages: boolean = true): Promise<void> {
-		if (!this.notebookKernelExists(cluster)) {
+		if (!this.jupyterKernelExists(cluster)) {
 			let notebookKernel: DatabricksKernel = new DatabricksKernel(cluster);
-			this.setKernel(this.getNotebookKernelName(cluster), notebookKernel);
+			this.setKernel(this.getJupyterKernelName(cluster), notebookKernel);
 			if (logMessages) {
 				ThisExtension.log(`Notebook Kernel for Databricks cluster '${cluster.cluster_id}' created!`)
 			}
@@ -101,9 +101,9 @@ export abstract class DatabricksKernelManager {
 			}
 		}
 
-		if (!this.interactiveKernelExists(cluster)) {
-			let interactiveKernel: DatabricksKernel = new DatabricksKernel(cluster, "interactive");
-			this.setKernel(this.getInteractiveKernelName(cluster), interactiveKernel);
+		if (!this.databricksKernelExists(cluster)) {
+			let interactiveKernel: DatabricksKernel = new DatabricksKernel(cluster, "databricks-notebook");
+			this.setKernel(this.getDatabricksKernelName(cluster), interactiveKernel);
 			if (logMessages) {
 				ThisExtension.log(`Interactive Kernel for Databricks cluster '${cluster.cluster_id}' created!`)
 			}
@@ -116,8 +116,8 @@ export abstract class DatabricksKernelManager {
 	}
 
 	static async removeKernels(cluster: iDatabricksCluster, logMessages: boolean = true): Promise<void> {
-		if (this.notebookKernelExists(cluster)) {
-			this.removeKernel(this.getNotebookKernelName(cluster));
+		if (this.jupyterKernelExists(cluster)) {
+			this.removeKernel(this.getJupyterKernelName(cluster));
 			if (logMessages) {
 				ThisExtension.log(`Notebook Kernel for Databricks cluster '${cluster.cluster_id}' removed!`)
 			}
@@ -128,8 +128,8 @@ export abstract class DatabricksKernelManager {
 			}
 		}
 
-		if (this.interactiveKernelExists(cluster)) {
-			this.removeKernel(this.getInteractiveKernelName(cluster));
+		if (this.databricksKernelExists(cluster)) {
+			this.removeKernel(this.getDatabricksKernelName(cluster));
 			if (logMessages) {
 				ThisExtension.log(`Interactive Kernel for Databricks cluster '${cluster.cluster_id}' removed!`)
 			}
@@ -142,16 +142,16 @@ export abstract class DatabricksKernelManager {
 	}
 
 	static async restartClusterKernel(cluster: iDatabricksCluster): Promise<void> {
-		let kernel: DatabricksKernel = this.getNotebookKernel(cluster)
+		let kernel: DatabricksKernel = this.getJupyterKernel(cluster)
 		if (kernel) {
 			kernel.restart();
 		}
 	}
 
-	static async restartNotebookKernel(notebook: { notebookEditor: { notebookUri: vscode.Uri } } | undefined | vscode.Uri): Promise<void> {
+	static async restartJupyterKernel(notebook: { notebookEditor: { notebookUri: vscode.Uri } } | undefined | vscode.Uri): Promise<void> {
 		let notebookUri: vscode.Uri = undefined;
 
-		ThisExtension.setStatusBar("Restarting Kernel ...", true);
+		ThisExtension.setStatusBar("Restarting Jupyter Kernel ...", true);
 
 		if (notebook instanceof vscode.Uri) {
 			notebookUri = notebook;
@@ -164,7 +164,7 @@ export abstract class DatabricksKernelManager {
 			kernel.restart(notebookUri);
 		}
 
-		ThisExtension.setStatusBar("Kernel restarted!");
+		ThisExtension.setStatusBar("Jupyter Kernel restarted!");
 	}
 
 	static async updateWidgets(notebook: { notebookEditor: { notebookUri: vscode.Uri } } | undefined | vscode.Uri): Promise<void> {