Sparkmagic Kerberos authentication issue

[greg-kan]

Hello.

I have been trying to set up kerberos authentication to Spark via Sparkmagic and Livy.
As mentioned in the documentation, I am getting a ticket, but the authentication is not working and I am getting a 401 "Authentication required" error.

Questions:

1. Maybe I inserted "kerberos_auth_configuration" in the wrong place of config.json (I tryied diffren places although)? What is the right place for the "kerberos_auth_configuration" section?

"authenticators": {
"Kerberos": "sparkmagic.auth.kerberos.Kerberos",
"kerberos_auth_configuration": {
"mutual_authentication": 1,
"service": "HTTP",
"delegate": false,
"force_preemptive": false,
"principal": "username@DOMAIN_REALM",
"sanitize_mutual_error_response": true,
"send_cbt": true
},
"None": "sparkmagic.auth.customauth.Authenticator",
"Basic_Access": "sparkmagic.auth.basic.Basic"
},

2. Is it possible to point to the kerberos ticket cache or to the keytab DIRECTLY? It seems that Sparkmagic does not see my ticket.

My client OS is Windows 10.
My config.json file here:

{
"kernel_python_credentials" : {
"username": "username@DOMAIN_REALM",
"url": "http://server_url:8999",
"auth": "Kerberos"
},

"kernel_scala_credentials" : {
"username": "username@DOMAIN_REALM",
"url": "http://server_url:8999",
"auth": "Kerberos"
},
"kernel_r_credentials": {
"username": "username@DOMAIN_REALM",
"url": "http://server_url:8999",
"auth": "Kerberos"
},

"logging_config": {
"version": 1,
"formatters": {
"magicsFormatter": {
"format": "%(asctime)s\t%(levelname)s\t%(message)s",
"datefmt": ""
}
},
"handlers": {
"magicsHandler": {
"class": "hdijupyterutils.filehandler.MagicsFileHandler",
"formatter": "magicsFormatter",
"home_path": "C:/Users/username/.sparkmagic"
}
},
"loggers": {
"magicsLogger": {
"handlers": ["magicsHandler"],
"level": "DEBUG",
"propagate": 0
}
}
},
"authenticators": {
"Kerberos": "sparkmagic.auth.kerberos.Kerberos",
"kerberos_auth_configuration": {
"mutual_authentication": 1,
"service": "HTTP",
"delegate": false,
"force_preemptive": false,
"principal": "username@DOMAIN_REALM",
"sanitize_mutual_error_response": true,
"send_cbt": true
},
"None": "sparkmagic.auth.customauth.Authenticator",
"Basic_Access": "sparkmagic.auth.basic.Basic"
},

"wait_for_idle_timeout_seconds": 15,
"livy_session_startup_timeout_seconds": 60,

"fatal_error_suggestion": "The code failed because of a fatal error:\n\t{}.\n\nSome things to try:\na) Make sure Spark has enough available resources for Jupyter to create a Spark context.\nb) Contact your Jupyter administrator to make sure the Spark magics library is configured correctly.\nc) Restart the kernel.",

"ignore_ssl_errors": false,

"session_configs": {
"driverMemory": "1000M",
"executorCores": 2
},

"use_auto_viz": true,
"coerce_dataframe": true,
"max_results_sql": 2500,
"pyspark_dataframe_encoding": "utf-8",

"heartbeat_refresh_seconds": 30,
"livy_server_heartbeat_timeout_seconds": 0,
"heartbeat_retry_seconds": 10,

"server_extension_default_kernel_name": "pysparkkernel",
"custom_headers": {},

"retry_policy": "configurable",
"retry_seconds_to_sleep_list": [0.2, 0.5, 1, 3, 5],
"configurable_retry_policy_max_retries": 8
}

[Armadik]

Livy server must be protected by SPNEGO. This is true?